--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:37:50.941745Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-20622-549955698d-5qphk to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:37:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:37:50Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cbf88106141d namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 9484b663-f459-4517-a9c3-f9d8918441f9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:51Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:37:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-20622-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:51Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cbf88cd9a0e2 namespace: kserve-ci-e2e-test resourceVersion: "19869" uid: bacb0275-eeba-44ff-937d-23be79cb8604 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:51Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:37:51Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:37:51Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cbf8bdf9cd4c namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 03c8e048-1ae3-4679-a7ba-3202aae54a83 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-20622} kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:37:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:51Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cbf8bf262bd4 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 16acfe55-1d71-4def-bed8-30cfdaedbe4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-20622} kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:37:52Z" message: 'Created container: ensemble-graph-20622' metadata: creationTimestamp: "2026-05-25T11:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:52Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cbf8c1ff96cb namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 1869f626-9975-4043-8a36-beb4696d42e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-20622} kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:37:52Z" message: Started container ensemble-graph-20622 metadata: creationTimestamp: "2026-05-25T11:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:52Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cbf8c2714078 namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 89a48e99-1de8-49eb-aca4-f286712fd97d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-20622} kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Stopping container ensemble-graph-20622 metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:05Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cc6bae84a4d1 namespace: kserve-ci-e2e-test resourceVersion: "23346" uid: ebfa975b-dff4-414f-bb47-e115a31070c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-20622} kind: Pod name: ensemble-graph-20622-549955698d-5qphk namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: d328ad6c-d586-4706-8021-e4876cd7c03c kind: Event lastTimestamp: "2026-05-25T11:46:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:33Z" name: ensemble-graph-20622-549955698d-5qphk.18b2cc6c3f80a7d0 namespace: kserve-ci-e2e-test resourceVersion: "23678" uid: 6d029af2-193d-4f32-b13a-399a180d6456 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-20622-549955698d namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: c0733570-7ae3-460c-921d-d28f2f201e7e kind: Event lastTimestamp: "2026-05-25T11:37:50Z" message: 'Created pod: ensemble-graph-20622-549955698d-5qphk' metadata: creationTimestamp: "2026-05-25T11:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:50Z" name: ensemble-graph-20622-549955698d.18b2cbf880652a7d namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 9b4301e2-1a63-46e0-b43f-0568856e2986 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-20622 namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: febfcfa6-8ecd-4eb6-b74f-2e5bef1cd888 kind: Event lastTimestamp: "2026-05-25T11:37:50Z" message: Scaled up replica set ensemble-graph-20622-549955698d from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:50Z" name: ensemble-graph-20622.18b2cbf87f407d30 namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: ffc056af-c5de-4133-9fc3-a1bab845155a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-20622 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 3e0f0d8b-1d45-43cc-ac3e-55235b0cfc57 kind: Event lastTimestamp: "2026-05-25T11:37:51Z" message: InferenceGraph [ensemble-graph-20622] is Ready metadata: creationTimestamp: "2026-05-25T11:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:51Z" name: ensemble-graph-20622.18b2cbf884e853b9 namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 3d094bfa-3333-40fb-882d-26b54aef3cc2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-20622 namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 3b655426-8d08-4328-b651-40a1b1a57531 kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:20Z" name: ensemble-graph-20622.18b2cbfbfd92def2 namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: a1c6c685-6552-4883-8bb5-380b6393c468 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-20622 namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 3b655426-8d08-4328-b651-40a1b1a57531 kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:20Z" name: ensemble-graph-20622.18b2cbfbfe011eff namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 1f275959-eb1e-4b48-9308-5203383eeae8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:36:40.636349Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18873" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-49543-6ccd8b5c87-lwpqk to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:36:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:36:40Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbe8227edf36 namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: 1bb153a9-34c0-47ea-acd3-401f88b41950 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:41Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18882" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: "2026-05-25T11:36:41Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:36:41Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbe83c23d5f0 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 95903bc4-0e3f-4212-921a-e2611813bcc9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49543} kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: "2026-05-25T11:36:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:41Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbe83d775c12 namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 5470e389-d82e-4ac8-a7d1-4e5ffc31e5e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49543} kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: "2026-05-25T11:36:41Z" message: 'Created container: ensemble-graph-49543' metadata: creationTimestamp: "2026-05-25T11:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:41Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbe8405200be namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: c031ad54-3a88-407f-a580-ff57f10e02a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49543} kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: "2026-05-25T11:36:41Z" message: Started container ensemble-graph-49543 metadata: creationTimestamp: "2026-05-25T11:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:41Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbe840c88312 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: 8cf687a9-28b7-41b7-b277-ab55fd6f55d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49543} kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Stopping container ensemble-graph-49543 metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:50Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbea7a78704f namespace: kserve-ci-e2e-test resourceVersion: "19000" uid: f2fdceac-61d5-45ad-9871-8ea2f97e0c3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49543} kind: Pod name: ensemble-graph-49543-6ccd8b5c87-lwpqk namespace: kserve-ci-e2e-test resourceVersion: "18874" uid: 99536bae-c9b2-4d1d-865a-f10ed90b9222 kind: Event lastTimestamp: "2026-05-25T11:37:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:17Z" name: ensemble-graph-49543-6ccd8b5c87-lwpqk.18b2cbeafa5e7161 namespace: kserve-ci-e2e-test resourceVersion: "19399" uid: a9653967-81cb-4b51-a295-afd98381df95 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-49543-6ccd8b5c87 namespace: kserve-ci-e2e-test resourceVersion: "18870" uid: 2bc53ded-5f12-4678-bbad-fc00c5d13b33 kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'Created pod: ensemble-graph-49543-6ccd8b5c87-lwpqk' metadata: creationTimestamp: "2026-05-25T11:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: ensemble-graph-49543-6ccd8b5c87.18b2cbe821fe6d17 namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: 8af47bdb-1291-4e4c-b3e0-68e987bda320 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-49543 namespace: kserve-ci-e2e-test resourceVersion: "18868" uid: cb44af49-b65d-49e7-9709-550196f4961a kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: Scaled up replica set ensemble-graph-49543-6ccd8b5c87 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: ensemble-graph-49543.18b2cbe8211c0ba4 namespace: kserve-ci-e2e-test resourceVersion: "18872" uid: 794eb603-59e1-47ad-8e1b-98cd362373ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-49543 namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 292e7030-1e65-45bb-8a45-c14eefb8af4e kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: InferenceGraph [ensemble-graph-49543] is Ready metadata: creationTimestamp: "2026-05-25T11:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:40Z" name: ensemble-graph-49543.18b2cbe827799255 namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 19a629bf-b34f-48c0-b0be-43e233b23c26 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:36:50.957232Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19076" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-20622-predictor-57456b66c5-zxs4z to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbea89ab1074 namespace: kserve-ci-e2e-test resourceVersion: "19080" uid: a447fd3d-64f1-4063-b7cc-90fe1fb29ec7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19086" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaa36ad484 namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: 6bd16b5c-6f1e-4268-8972-b3d43f41305e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaa4f17a86 namespace: kserve-ci-e2e-test resourceVersion: "19106" uid: 1b7471fb-642a-40f4-8d8e-74d10a73e7b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaa7a82a35 namespace: kserve-ci-e2e-test resourceVersion: "19107" uid: d974677c-1951-4c02-bfbf-f141064e3a9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaa822b26d namespace: kserve-ci-e2e-test resourceVersion: "19108" uid: 2bf38f72-2d49-4c36-afa9-a5910ff416c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaa82f96cd namespace: kserve-ci-e2e-test resourceVersion: "19109" uid: 44f1e417-a4c4-44d1-a4f4-a815ecf80c2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaaacb0af9 namespace: kserve-ci-e2e-test resourceVersion: "19110" uid: 8245e6ec-b937-4cd8-8921-5cee349a1c09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeaab399a6f namespace: kserve-ci-e2e-test resourceVersion: "19111" uid: 8ac00fee-ad6f-4a3b-b307-d1ab7fbc4f79 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T11:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:46:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:08Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cbeafc91f42f namespace: kserve-ci-e2e-test resourceVersion: "23490" uid: 4975a2f2-a4d8-4857-a873-eeecf3ad5917 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cc6bba36d0dc namespace: kserve-ci-e2e-test resourceVersion: "23397" uid: db218368-36c2-4c45-aa23-0ccabfa034c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cc6bba37772c namespace: kserve-ci-e2e-test resourceVersion: "23402" uid: 627f6ec5-252d-420b-8e5c-4359f197a8e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20622-predictor-57456b66c5-zxs4z namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: 6d4b495f-f2db-40db-a7bb-e9b722a5101c kind: Event lastTimestamp: "2026-05-25T11:46:08Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:08Z" name: error-404-isvc-20622-predictor-57456b66c5-zxs4z.18b2cc6c71191ae1 namespace: kserve-ci-e2e-test resourceVersion: "23489" uid: 99c594f8-a046-418f-bb10-18e62f3a9aed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-20622-predictor-57456b66c5 namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: b395170d-4bf7-4aa6-8325-7af939884d3a kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: 'Created pod: error-404-isvc-20622-predictor-57456b66c5-zxs4z' metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-20622-predictor-57456b66c5.18b2cbea89418935 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: 9e787905-6354-4920-a14f-edd99df2d0f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 8617d91c-f943-482d-9352-2edf6a89b084 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Scaled up replica set error-404-isvc-20622-predictor-57456b66c5 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-20622-predictor.18b2cbea878bd7c0 namespace: kserve-ci-e2e-test resourceVersion: "19071" uid: 6f6cdd68-6459-4614-af6d-0afff5d4781f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 2c12cb91-c415-4801-a286-4cb205700481 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-20622-predictor.18b2cbee0620ec79 namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 9c46858c-590f-48f7-92c7-5be515df6d59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 2c12cb91-c415-4801-a286-4cb205700481 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-20622-predictor.18b2cbee0691fb73 namespace: kserve-ci-e2e-test resourceVersion: "19544" uid: ed3b4e55-4db2-4d57-a54e-cd81f3042e94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19267" uid: 2c12cb91-c415-4801-a286-4cb205700481 kind: Event lastTimestamp: "2026-05-25T11:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:05Z" name: error-404-isvc-20622-predictor.18b2cbf504200381 namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 32f94b27-a3b4-452a-b69f-948af1655c2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19267" uid: 2c12cb91-c415-4801-a286-4cb205700481 kind: Event lastTimestamp: "2026-05-25T11:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:05Z" name: error-404-isvc-20622-predictor.18b2cbf5046915b8 namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: c7cdde91-97e7-4aec-8ba6-2adb885b363d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 53759e9a-91f3-4813-b6b7-147ffae5e117 kind: Event lastTimestamp: "2026-05-25T11:37:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:39Z" name: error-404-isvc-20622.18b2cbea87613897 namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b53d4239-7328-4c10-9255-47eef2f9300e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 53759e9a-91f3-4813-b6b7-147ffae5e117 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-20622": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-20622": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-20622.18b2cbea87b7afb8 namespace: kserve-ci-e2e-test resourceVersion: "19073" uid: 9cb80f4d-75c1-429c-9fc7-9e2c78f70593 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 53759e9a-91f3-4813-b6b7-147ffae5e117 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-20622": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-20622.18b2cbea87b7f5dc namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: c8e141d3-fabc-4208-993a-77176a76c848 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 53759e9a-91f3-4813-b6b7-147ffae5e117 kind: Event lastTimestamp: "2026-05-25T11:37:39Z" message: InferenceService [error-404-isvc-20622] is Ready metadata: creationTimestamp: "2026-05-25T11:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:39Z" name: error-404-isvc-20622.18b2cbf5ba4eb0c6 namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: d4d3fa2b-570d-40ad-a5c2-ebee0f106136 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:24:55.390638Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12749" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-21751-predictor-6888887675-wtlk4 to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:24:55Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb43ee934bcc namespace: kserve-ci-e2e-test resourceVersion: "12753" uid: abf87833-15a7-42b2-8096-5971ff32a475 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12759" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:24:55Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb440806a6bd namespace: kserve-ci-e2e-test resourceVersion: "12786" uid: 6451be20-1865-4008-aedf-7a3b4fef30c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:55Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb4409768e34 namespace: kserve-ci-e2e-test resourceVersion: "12788" uid: e0979f9d-8bbf-4fe7-a3a3-1004ee225d47 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:07Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" in 11.574s (11.574s including waiting). Image size: 1334724716 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:07Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb46bb564848 namespace: kserve-ci-e2e-test resourceVersion: "12944" uid: 9facc553-c777-4595-86e5-bcee41a20efd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:07Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb46be8905be namespace: kserve-ci-e2e-test resourceVersion: "12945" uid: 10830ebf-d721-466f-b2c1-dcc49d69ead1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:07Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb46bef53671 namespace: kserve-ci-e2e-test resourceVersion: "12946" uid: 38047b0b-7fe9-47e2-ab34-ab73a1510145 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T11:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:07Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb46bf0115ca namespace: kserve-ci-e2e-test resourceVersion: "12947" uid: de58ebae-ec63-4477-8327-31ee76ef8270 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:09Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.308s (2.308s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:09Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb4748a081c8 namespace: kserve-ci-e2e-test resourceVersion: "12969" uid: 7cfe2bc6-c7e7-487a-ae8a-48a7d3d35279 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:09Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb474c02c6da namespace: kserve-ci-e2e-test resourceVersion: "12970" uid: d2e410d6-7cf3-419f-99b5-b8f85a200808 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:25:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:09Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb474c7c6da5 namespace: kserve-ci-e2e-test resourceVersion: "12971" uid: 581b43d5-8d0c-4d57-8092-dfb5fccfa742 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb477542d7ff namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: 0b28b708-0c1b-472d-b8c8-689fe16bfe73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:25Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb58e17be42d namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: 2c9c429d-6137-4683-857a-2abab3e9754f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:25Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb58e17c83b5 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: b15ad76e-8936-4e19-8067-0e67de52f04f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21751-predictor-6888887675-wtlk4 namespace: kserve-ci-e2e-test resourceVersion: "12751" uid: c63d88a9-9acf-41e8-a675-83742a1f2064 kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-21751-predictor-6888887675-wtlk4.18b2cb5927573882 namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: 7b0d1715-944a-4763-8cd7-630c2c2bd5b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-21751-predictor-6888887675 namespace: kserve-ci-e2e-test resourceVersion: "12731" uid: c33f5e8c-60e2-4f13-91e1-7c344865b9ac kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: 'Created pod: error-404-isvc-21751-predictor-6888887675-wtlk4' metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: error-404-isvc-21751-predictor-6888887675.18b2cb43edfa8842 namespace: kserve-ci-e2e-test resourceVersion: "12750" uid: 788a1b4d-7dd1-4778-9ada-55c098b74f36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12728" uid: f2cfd36b-ca7c-4235-a48b-377d8b00012e kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Scaled up replica set error-404-isvc-21751-predictor-6888887675 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: error-404-isvc-21751-predictor.18b2cb43e75ddbd9 namespace: kserve-ci-e2e-test resourceVersion: "12737" uid: 9f72a233-74ad-4780-84bf-d26a6f61ebf9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12730" uid: 75c2bfc0-d717-4834-b499-9b7711be57bf kind: Event lastTimestamp: "2026-05-25T11:25:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:25Z" name: error-404-isvc-21751-predictor.18b2cb4765fb4277 namespace: kserve-ci-e2e-test resourceVersion: "13135" uid: f7c64ae8-c2f8-4247-b3a5-95da81a2f73e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12730" uid: 75c2bfc0-d717-4834-b499-9b7711be57bf kind: Event lastTimestamp: "2026-05-25T11:25:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:25Z" name: error-404-isvc-21751-predictor.18b2cb47665b8465 namespace: kserve-ci-e2e-test resourceVersion: "13136" uid: d35942b5-efdb-470c-9e3f-9b029ed86ecb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12985" uid: 75c2bfc0-d717-4834-b499-9b7711be57bf kind: Event lastTimestamp: "2026-05-25T11:26:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:10Z" name: error-404-isvc-21751-predictor.18b2cb4e63f31551 namespace: kserve-ci-e2e-test resourceVersion: "13528" uid: a0f5d378-2f7b-4101-9596-e734af5f38f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12985" uid: 75c2bfc0-d717-4834-b499-9b7711be57bf kind: Event lastTimestamp: "2026-05-25T11:26:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:10Z" name: error-404-isvc-21751-predictor.18b2cb4e647e6324 namespace: kserve-ci-e2e-test resourceVersion: "13529" uid: 2dbbfdbb-baff-4241-a4d3-08e8b7e81d00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21751 namespace: kserve-ci-e2e-test resourceVersion: "12720" uid: 872939d3-d23d-4532-9c28-5df4809cf11c kind: Event lastTimestamp: "2026-05-25T11:25:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:25:56Z" name: error-404-isvc-21751.18b2cb43e71fd211 namespace: kserve-ci-e2e-test resourceVersion: "13415" uid: 274b148a-b659-494d-9042-849bf949f521 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21751 namespace: kserve-ci-e2e-test resourceVersion: "13413" uid: 872939d3-d23d-4532-9c28-5df4809cf11c kind: Event lastTimestamp: "2026-05-25T11:25:56Z" message: InferenceService [error-404-isvc-21751] is Ready metadata: creationTimestamp: "2026-05-25T11:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:25:56Z" name: error-404-isvc-21751.18b2cb5230d632a0 namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: 29c1850d-1ce2-4ec2-9178-72247a9ec50e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:36:10.263684Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18543" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe11024b6fd namespace: kserve-ci-e2e-test resourceVersion: "18547" uid: 06b1b6d5-de81-4a1f-942b-7d27e1e4de5a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-412bf-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe11d97a92f namespace: kserve-ci-e2e-test resourceVersion: "18560" uid: 90f2c6da-eaa3-4fca-8711-9e878d9b5417 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18557" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe14de64533 namespace: kserve-ci-e2e-test resourceVersion: "18587" uid: ecda83e8-05cf-4f68-832c-697fe65dc1c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe14fbe0a9d namespace: kserve-ci-e2e-test resourceVersion: "18590" uid: cb1756bd-11c9-4c8d-a9cd-3c5fb4f54ed3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe153ed04ea namespace: kserve-ci-e2e-test resourceVersion: "18591" uid: 4acd999e-e503-452e-950c-5ecaec850e27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe1546186c9 namespace: kserve-ci-e2e-test resourceVersion: "18592" uid: 0b13c103-7fe5-4fd9-8fb6-73d0d81c7caa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe1546d2f10 namespace: kserve-ci-e2e-test resourceVersion: "18593" uid: 6c3e3773-490d-41db-bc45-2852256341c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe1580359c7 namespace: kserve-ci-e2e-test resourceVersion: "18594" uid: 628ce6a9-c11a-434b-b6e4-18d94a90f46c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe15873fb6e namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: a085fb89-02f9-45b1-9791-c71123d2e46f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:36:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:48Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbe1a3b8f9df namespace: kserve-ci-e2e-test resourceVersion: "18966" uid: f04a1c8b-da28-4a9a-b0e8-f492cc67076b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbf1654b7023 namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: a8450f30-5ffe-4293-b2ab-9e6b9b0ad55b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbf1654c00e0 namespace: kserve-ci-e2e-test resourceVersion: "19512" uid: 6c7ec873-5baf-4b98-9325-1bc4d1ff7db0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4 namespace: kserve-ci-e2e-test resourceVersion: "18545" uid: be304323-b880-4e46-b36b-a4b1c9711750 kind: Event lastTimestamp: "2026-05-25T11:37:23Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:23Z" name: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4.18b2cbf22be6f1b5 namespace: kserve-ci-e2e-test resourceVersion: "19595" uid: 0f007122-5a7e-4a24-ac26-a695ba815e6a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-412bf-predictor-bd476c8cc namespace: kserve-ci-e2e-test resourceVersion: "18534" uid: d3f054f5-67db-4095-be99-dde92c91bdd5 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'Created pod: error-404-isvc-412bf-predictor-bd476c8cc-gg7d4' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-412bf-predictor-bd476c8cc.18b2cbe10f22ec19 namespace: kserve-ci-e2e-test resourceVersion: "18544" uid: 42615d7d-5bdf-49fc-be5a-84e06e5a5c3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18531" uid: 36ccfb47-4277-448a-8f33-370025aee551 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: Scaled up replica set error-404-isvc-412bf-predictor-bd476c8cc from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-412bf-predictor.18b2cbe10c4fcf78 namespace: kserve-ci-e2e-test resourceVersion: "18536" uid: 85900262-f140-4e53-ac28-0c15a1639031 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18533" uid: e828455a-f17c-46e3-8f4c-68291019264f kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: error-404-isvc-412bf-predictor.18b2cbe48b17cccb namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 6241aa0e-b0b5-48d9-b26e-66547d5cabd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18533" uid: e828455a-f17c-46e3-8f4c-68291019264f kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: error-404-isvc-412bf-predictor.18b2cbe48b8f7d0a namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: bc2ed1a8-bfbf-4d31-bda5-a650e268a3cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: e828455a-f17c-46e3-8f4c-68291019264f kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:10Z" name: error-404-isvc-412bf-predictor.18b2cbeb8919bcd1 namespace: kserve-ci-e2e-test resourceVersion: "19331" uid: a2b02b30-416b-4fc7-a3fb-e8c9a3cdf6df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: e828455a-f17c-46e3-8f4c-68291019264f kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:10Z" name: error-404-isvc-412bf-predictor.18b2cbeb898479dc namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: 811c3dd8-f812-4343-886d-e53080fd5bf5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "18507" uid: e27ce9a2-2e90-4a3e-b2e7-49d1821c6914 kind: Event lastTimestamp: "2026-05-25T11:36:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:58Z" name: error-404-isvc-412bf.18b2cbe10c522916 namespace: kserve-ci-e2e-test resourceVersion: "19210" uid: f6fe786f-96b5-4f8d-9a64-e60f6fed00f7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "18507" uid: e27ce9a2-2e90-4a3e-b2e7-49d1821c6914 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-412bf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-412bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-412bf.18b2cbe10cefd9b3 namespace: kserve-ci-e2e-test resourceVersion: "18540" uid: 16f50905-7f4a-415d-b237-f0505d7fc22b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "18507" uid: e27ce9a2-2e90-4a3e-b2e7-49d1821c6914 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-412bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-412bf.18b2cbe10cf025a0 namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: 40e8319b-c071-480c-8506-389e253bfd17 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: e27ce9a2-2e90-4a3e-b2e7-49d1821c6914 kind: Event lastTimestamp: "2026-05-25T11:36:58Z" message: InferenceService [error-404-isvc-412bf] is Ready metadata: creationTimestamp: "2026-05-25T11:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:58Z" name: error-404-isvc-412bf.18b2cbec5f75820a namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: d3e8653f-72a6-473d-83d0-c094914a2730 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:35:41.085347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18176" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-49543-predictor-585f55b477-zzzff to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda44faa8de namespace: kserve-ci-e2e-test resourceVersion: "18183" uid: 3a41f19a-4faa-442c-bb30-6dc6e08bac11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18188" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda5ea714f4 namespace: kserve-ci-e2e-test resourceVersion: "18209" uid: 1a1c7e3a-30e8-4a21-bbdb-5ccc115c723d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda60af090d namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: 982a6f8d-7532-43d2-85f1-d58b7f757aec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda63b53adc namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: a5c4c9bd-8215-4f1a-964b-d7c59f885d9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda6426e4e4 namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: 4f200e45-3996-4c73-a0ab-7722187403c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda6432fb7b namespace: kserve-ci-e2e-test resourceVersion: "18214" uid: 5cce6101-b359-4846-a8b2-a7d797f3992e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda67de6b10 namespace: kserve-ci-e2e-test resourceVersion: "18216" uid: cdf2cca7-82d2-45c9-8151-20b3b5eb56f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbda6851fc67 namespace: kserve-ci-e2e-test resourceVersion: "18217" uid: 4b63f370-f14f-4246-949d-66ec88b00a05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:36:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:19Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbdadcf690da namespace: kserve-ci-e2e-test resourceVersion: "18676" uid: df2aac16-d6e0-4bd6-b36c-c135976a6880 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbea8653b9a1 namespace: kserve-ci-e2e-test resourceVersion: "19064" uid: 118cd449-4f32-4b1a-ba7c-3db555c929e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49543-predictor-585f55b477-zzzff namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: a8bf0f51-c69e-4b1d-bcd9-9f5a4bd9a28a kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:50Z" name: error-404-isvc-49543-predictor-585f55b477-zzzff.18b2cbea8656b76c namespace: kserve-ci-e2e-test resourceVersion: "19067" uid: e39836c2-287f-4e33-98ff-9276ba2480f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-49543-predictor-585f55b477 namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: 61572147-68d3-4678-b1ff-3d430714a4bd kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: 'Created pod: error-404-isvc-49543-predictor-585f55b477-zzzff' metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-49543-predictor-585f55b477.18b2cbda43035ec3 namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: cecb37d7-6e9d-4de6-bc30-82f4ffb5d13a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18148" uid: 5f417d4b-b7d5-4ba5-a398-37162545867f kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: Scaled up replica set error-404-isvc-49543-predictor-585f55b477 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:35:40Z" name: error-404-isvc-49543-predictor.18b2cbda3ca29c12 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 583266c1-0ae9-474a-aff5-8755c2723501 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:35:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18152" uid: 3357922a-7340-4eb6-a3b1-183481fc634d kind: Event lastTimestamp: "2026-05-25T11:36:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:26Z" name: error-404-isvc-49543-predictor.18b2cbddbb38b895 namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: 3e7ed095-d7ce-4b63-a839-da5ca6b0ba43 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:35:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18152" uid: 3357922a-7340-4eb6-a3b1-183481fc634d kind: Event lastTimestamp: "2026-05-25T11:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:26Z" name: error-404-isvc-49543-predictor.18b2cbddbbb1f458 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: c7ebf5ee-caa9-4854-93c9-95283bedb781 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18344" uid: 3357922a-7340-4eb6-a3b1-183481fc634d kind: Event lastTimestamp: "2026-05-25T11:36:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:41Z" name: error-404-isvc-49543-predictor.18b2cbe13aa52d49 namespace: kserve-ci-e2e-test resourceVersion: "18897" uid: 28be87a3-5369-4f24-a649-78537892e300 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18344" uid: 3357922a-7340-4eb6-a3b1-183481fc634d kind: Event lastTimestamp: "2026-05-25T11:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:41Z" name: error-404-isvc-49543-predictor.18b2cbe13b2b4e27 namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: d2225877-5225-42f3-8c93-35aad0b6fb0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49543 namespace: kserve-ci-e2e-test resourceVersion: "18134" uid: 5d91921f-f010-4d28-a6f7-b3b6bbc81d7e kind: Event lastTimestamp: "2026-05-25T11:36:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:29Z" name: error-404-isvc-49543.18b2cbda3c5b7970 namespace: kserve-ci-e2e-test resourceVersion: "18779" uid: f5bb2627-b6a6-4d81-a336-682bcf32db04 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49543 namespace: kserve-ci-e2e-test resourceVersion: "18777" uid: 5d91921f-f010-4d28-a6f7-b3b6bbc81d7e kind: Event lastTimestamp: "2026-05-25T11:36:29Z" message: InferenceService [error-404-isvc-49543] is Ready metadata: creationTimestamp: "2026-05-25T11:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:29Z" name: error-404-isvc-49543.18b2cbe59a8f810b namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 06120854-62de-4044-921f-e34af2eee50b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:26:25.617463Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13803" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-55329-predictor-689f64f8cf-7x2km to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:26:25Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb58f0837b4a namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: 65de3646-d347-4c0e-b97a-ffb9e6701547 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13811" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb5909ce3459 namespace: kserve-ci-e2e-test resourceVersion: "13820" uid: cb14fcc5-1919-4171-9706-80d6c04a5d63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb590b3862d7 namespace: kserve-ci-e2e-test resourceVersion: "13822" uid: dd96ce05-bef4-4f54-81fb-9fffe1beb66d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb590e1bc3c7 namespace: kserve-ci-e2e-test resourceVersion: "13824" uid: 38a73007-c07f-46f7-abb3-7f4ab03755c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb590e7e602c namespace: kserve-ci-e2e-test resourceVersion: "13825" uid: 651bc9c8-0746-4a4f-8735-fa990ecca2c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb590e8b8e6e namespace: kserve-ci-e2e-test resourceVersion: "13826" uid: 900ca26a-895b-4736-944f-0eb67db3fe76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb59114caf3b namespace: kserve-ci-e2e-test resourceVersion: "13827" uid: 199966fc-0954-4718-8755-41d5261a327d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb5911b0bac9 namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: ae83daf0-da40-4b44-85cc-2209006bb286 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:35:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:42Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cb59364a37d5 namespace: kserve-ci-e2e-test resourceVersion: "18237" uid: 189148ff-b133-4a08-911e-3984381a40cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cbda45a36b81 namespace: kserve-ci-e2e-test resourceVersion: "18185" uid: d2d1ee9c-64e1-4d9b-8b6c-0c0ccf3396ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cbda45a446be namespace: kserve-ci-e2e-test resourceVersion: "18186" uid: 9098a4a5-eca1-4963-902f-d07baff204c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55329-predictor-689f64f8cf-7x2km namespace: kserve-ci-e2e-test resourceVersion: "13805" uid: 47d7ff78-3506-4b71-931a-1b9788def94b kind: Event lastTimestamp: "2026-05-25T11:35:42Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:42Z" name: error-404-isvc-55329-predictor-689f64f8cf-7x2km.18b2cbdaaa9e68e6 namespace: kserve-ci-e2e-test resourceVersion: "18235" uid: 258fc491-037a-4529-9c83-cb5b434717ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-55329-predictor-689f64f8cf namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 67bd1a22-55f6-4b64-a689-ad51e6e360a6 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: 'Created pod: error-404-isvc-55329-predictor-689f64f8cf-7x2km' metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:25Z" name: error-404-isvc-55329-predictor-689f64f8cf.18b2cb58f0180c8e namespace: kserve-ci-e2e-test resourceVersion: "13804" uid: c895fb12-15be-4da8-9f55-1c7bbfde342b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13791" uid: 3e00ca7e-2391-4b1b-92d8-541418b30eee kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Scaled up replica set error-404-isvc-55329-predictor-689f64f8cf from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:25Z" name: error-404-isvc-55329-predictor.18b2cb58e83b0556 namespace: kserve-ci-e2e-test resourceVersion: "13795" uid: 2931dbf0-c01b-46d9-b308-bab97fcff471 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: f1423be3-c124-4510-93cc-6fe42cfeee32 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-55329-predictor.18b2cb5c66d95083 namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: f0b5b7ce-7ad4-4efd-9aba-34cff2ed0a86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: f1423be3-c124-4510-93cc-6fe42cfeee32 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-55329-predictor.18b2cb5c6742e06a namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: b4d0c820-2744-43ac-b375-96b2f1a3672b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: f1423be3-c124-4510-93cc-6fe42cfeee32 kind: Event lastTimestamp: "2026-05-25T11:27:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:40Z" name: error-404-isvc-55329-predictor.18b2cb6364cf417d namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: a26372e3-54c8-4634-9b25-967c4417e5e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: f1423be3-c124-4510-93cc-6fe42cfeee32 kind: Event lastTimestamp: "2026-05-25T11:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:40Z" name: error-404-isvc-55329-predictor.18b2cb636542d913 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: bbc2c21c-d635-4d53-ad26-417e481b831a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: f1423be3-c124-4510-93cc-6fe42cfeee32 kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: deployments.apps "error-404-isvc-55329-predictor" not found metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:35:40Z" name: error-404-isvc-55329-predictor.18b2cbda3df86d2a namespace: kserve-ci-e2e-test resourceVersion: "18164" uid: 58668619-b455-4fbd-8a20-237a6eba53c1 reason: FailedGetScale reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: f1423be3-c124-4510-93cc-6fe42cfeee32 kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: 'Operation cannot be fulfilled on horizontalpodautoscalers.autoscaling "error-404-isvc-55329-predictor": StorageError: invalid object, Code: 4, Key: /kubernetes.io/horizontalpodautoscalers/kserve-ci-e2e-test/error-404-isvc-55329-predictor, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: f1423be3-c124-4510-93cc-6fe42cfeee32, UID in object meta: ' metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:35:40Z" name: error-404-isvc-55329-predictor.18b2cbda3ea37217 namespace: kserve-ci-e2e-test resourceVersion: "18168" uid: fcfe07da-b3cf-4efb-8108-5ca24e12546a reason: FailedUpdateStatus reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-55329 namespace: kserve-ci-e2e-test resourceVersion: "13776" uid: fefd48b6-8137-4693-bae3-63ed9a2a4a0c kind: Event lastTimestamp: "2026-05-25T11:27:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:12Z" name: error-404-isvc-55329.18b2cb58e8086500 namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: f45f2d53-feba-4b32-a052-c198e7232403 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-55329 namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: fefd48b6-8137-4693-bae3-63ed9a2a4a0c kind: Event lastTimestamp: "2026-05-25T11:27:12Z" message: InferenceService [error-404-isvc-55329] is Ready metadata: creationTimestamp: "2026-05-25T11:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:12Z" name: error-404-isvc-55329.18b2cb63f2d8dedb namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: 78b2f1e3-b6a3-4934-b139-20eb57d62387 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:37:20.454846Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19517" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf167dce811 namespace: kserve-ci-e2e-test resourceVersion: "19521" uid: 4572a9e7-90c3-4f4b-a7ca-031831b749a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b59e4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1759ea1fc namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: 6b0d3687-7b29-4476-a349-d9f5ae08a592 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1a568cbe4 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 5062fa77-ca27-4d2a-96b7-b853af4497cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1a6cefb05 namespace: kserve-ci-e2e-test resourceVersion: "19561" uid: d4c76df2-72b8-4a13-9033-a8a6d9dc99d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1aa286b48 namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: b149295b-e54f-4004-8383-9b2a83f323bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1aa99beb5 namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: 7bbb9a9f-cb6d-44d0-8850-8840aba5d847 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1aaa535a0 namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: e1206fb6-5aef-4330-ab42-50d674ea9340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1ae87148a namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: b0d46c9c-e8ec-448a-9ebd-4f36f542b65b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1af4e1e1a namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: 23c145f3-0d32-4aaa-a66b-4ce477843580 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:46:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:38Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cbf1c32839a8 namespace: kserve-ci-e2e-test resourceVersion: "23862" uid: f02c0b25-17c2-4a3e-8db1-867b08954134 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cc729237ba06 namespace: kserve-ci-e2e-test resourceVersion: "23787" uid: 5567c4ac-0609-4cad-875c-bc7905b1b283 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cc7292381475 namespace: kserve-ci-e2e-test resourceVersion: "23789" uid: 7084f547-8bd7-4891-afae-a3f1edac5783 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 4919c573-90ea-47be-8e87-88291649d15b kind: Event lastTimestamp: "2026-05-25T11:46:37Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:37Z" name: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj.18b2cc733787660f namespace: kserve-ci-e2e-test resourceVersion: "23861" uid: 6aa27ab6-7f15-46a6-a2e8-abdc33496936 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b59e4-predictor-69f5bfb789 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: c2533c36-84fa-49a7-bc2a-c67691cc6e5e kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'Created pod: error-404-isvc-b59e4-predictor-69f5bfb789-d9zqj' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-b59e4-predictor-69f5bfb789.18b2cbf167716ffc namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: bec29091-8384-4e84-8496-5af3039c2bb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: 82927bbe-22fe-4afa-b4cf-20d42a3bebda kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Scaled up replica set error-404-isvc-b59e4-predictor-69f5bfb789 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-b59e4-predictor.18b2cbf1658ae348 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 5117f1c0-f8e6-4d9b-890e-3f222add833a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: f22b013c-fe59-4b05-9fd3-63fe02b42dbd kind: Event lastTimestamp: "2026-05-25T11:37:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:50Z" name: error-404-isvc-b59e4-predictor.18b2cbf4e40ffa08 namespace: kserve-ci-e2e-test resourceVersion: "19826" uid: 5866557a-9665-4e0e-9013-9580f20568bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: f22b013c-fe59-4b05-9fd3-63fe02b42dbd kind: Event lastTimestamp: "2026-05-25T11:37:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:50Z" name: error-404-isvc-b59e4-predictor.18b2cbf4e478af6e namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: 156ade87-7629-434a-a4f1-b2c76a2fc88a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: f22b013c-fe59-4b05-9fd3-63fe02b42dbd kind: Event lastTimestamp: "2026-05-25T11:38:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:35Z" name: error-404-isvc-b59e4-predictor.18b2cbfbe2145a3b namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: bd533b5c-2c2a-4d90-9628-e679e2a51b10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: f22b013c-fe59-4b05-9fd3-63fe02b42dbd kind: Event lastTimestamp: "2026-05-25T11:38:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:35Z" name: error-404-isvc-b59e4-predictor.18b2cbfbe27e384c namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: bc2bc12b-9478-4888-b7cc-dc7790a19d7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 14a2ee5e-0307-440c-83cd-8e6d4e7352c3 kind: Event lastTimestamp: "2026-05-25T11:38:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:38:08Z" name: error-404-isvc-b59e4.18b2cbf165556cd6 namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 9e874214-4b88-4b29-9a0a-54d847b9d3be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 14a2ee5e-0307-440c-83cd-8e6d4e7352c3 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-b59e4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b59e4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-b59e4.18b2cbf165aae1ec namespace: kserve-ci-e2e-test resourceVersion: "19514" uid: 3612488b-0bcc-4578-9d0a-df1bc81fd202 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 14a2ee5e-0307-440c-83cd-8e6d4e7352c3 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b59e4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:20Z" name: error-404-isvc-b59e4.18b2cbf165ab3b0f namespace: kserve-ci-e2e-test resourceVersion: "19515" uid: 5963ed26-a454-4daa-a8a2-b850334ab44d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 14a2ee5e-0307-440c-83cd-8e6d4e7352c3 kind: Event lastTimestamp: "2026-05-25T11:38:08Z" message: InferenceService [error-404-isvc-b59e4] is Ready metadata: creationTimestamp: "2026-05-25T11:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:38:08Z" name: error-404-isvc-b59e4.18b2cbfc7e587c15 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: ca0f9342-d024-4242-907d-20a85ed6d4b8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:46:35.255046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23791" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc7294882380 namespace: kserve-ci-e2e-test resourceVersion: "23795" uid: db85ed35-69cf-4def-991d-e28d77b2a1be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c1cab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72a1176849 namespace: kserve-ci-e2e-test resourceVersion: "23804" uid: 692f2150-4ed0-48e0-9935-be9a679be360 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23801" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72d2066337 namespace: kserve-ci-e2e-test resourceVersion: "23831" uid: e9017453-f38b-4ba2-82ac-fa7a8ffee985 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72d3a8702b namespace: kserve-ci-e2e-test resourceVersion: "23834" uid: 380c5ab6-d1f7-4d17-a901-664969eba32f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72d77ada50 namespace: kserve-ci-e2e-test resourceVersion: "23836" uid: 4162cb46-64e4-4d27-827c-584efa6ccb3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72d7ebd3e2 namespace: kserve-ci-e2e-test resourceVersion: "23837" uid: 065a7e64-984f-4c1c-af80-44f24931c459 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72d7f8e0c5 namespace: kserve-ci-e2e-test resourceVersion: "23838" uid: 1a0a337b-8ac2-4bd2-826d-968bf7ccffc8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72dbc2953f namespace: kserve-ci-e2e-test resourceVersion: "23839" uid: c4f74510-5291-4ae0-9d7a-bff1a8945b83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:46:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:36Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc72dc49a282 namespace: kserve-ci-e2e-test resourceVersion: "23840" uid: 7560ed74-4ade-4ee9-adfc-3e3a22b81a50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T11:47:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:13Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cc732fa68577 namespace: kserve-ci-e2e-test resourceVersion: "24214" uid: 9ac110de-a794-4473-a8ed-4de03b250cc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T12:03:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:03:54Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cd648e686164 namespace: kserve-ci-e2e-test resourceVersion: "31792" uid: 124e2134-bb74-4dfb-ad1c-6431f439b4b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk namespace: kserve-ci-e2e-test resourceVersion: "23793" uid: 3ee3c379-e619-4591-bd6a-62de2dc77a02 kind: Event lastTimestamp: "2026-05-25T12:03:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:03:54Z" name: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk.18b2cd648e68a61a namespace: kserve-ci-e2e-test resourceVersion: "31793" uid: 637b56ba-fee4-477e-a705-52ce01c62c99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c1cab-predictor-6c58cc8979 namespace: kserve-ci-e2e-test resourceVersion: "23779" uid: 06b9b803-5632-4a73-9c4c-7aa4601d7804 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'Created pod: error-404-isvc-c1cab-predictor-6c58cc8979-mf6jk' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c1cab-predictor-6c58cc8979.18b2cc7293f40d11 namespace: kserve-ci-e2e-test resourceVersion: "23792" uid: dd537b95-14eb-444b-9233-5baec49f8d67 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23773" uid: 5c05a6aa-5960-441c-ae16-18551e2aeec3 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Scaled up replica set error-404-isvc-c1cab-predictor-6c58cc8979 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c1cab-predictor.18b2cc7291a4bf8a namespace: kserve-ci-e2e-test resourceVersion: "23784" uid: 8276b5c1-f190-44f1-846a-382914796e95 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23778" uid: 6f54067e-9447-4495-8ff1-91d7382c045a kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: error-404-isvc-c1cab-predictor.18b2cc761067ebc7 namespace: kserve-ci-e2e-test resourceVersion: "24102" uid: 8166a9d0-15a0-4840-9405-95f14656a15e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23778" uid: 6f54067e-9447-4495-8ff1-91d7382c045a kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: error-404-isvc-c1cab-predictor.18b2cc7610f8da62 namespace: kserve-ci-e2e-test resourceVersion: "24103" uid: d1a310ce-d788-4646-b17b-26a7a9fb7631 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23966" uid: 6f54067e-9447-4495-8ff1-91d7382c045a kind: Event lastTimestamp: "2026-05-25T11:47:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:50Z" name: error-404-isvc-c1cab-predictor.18b2cc7d0eca2fa7 namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: fd22b455-e656-4705-ab71-6524db731e9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23966" uid: 6f54067e-9447-4495-8ff1-91d7382c045a kind: Event lastTimestamp: "2026-05-25T11:47:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:50Z" name: error-404-isvc-c1cab-predictor.18b2cc7d0f406a58 namespace: kserve-ci-e2e-test resourceVersion: "24711" uid: 893129ab-bf89-4b0a-838b-38347b023eb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "23749" uid: 52544e43-805b-4ef3-906c-bc9287317ac3 kind: Event lastTimestamp: "2026-05-25T11:47:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:23Z" name: error-404-isvc-c1cab.18b2cc729179aaf6 namespace: kserve-ci-e2e-test resourceVersion: "24452" uid: 444ff580-2b9d-40b2-a3e3-e4888b19ac9c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "23749" uid: 52544e43-805b-4ef3-906c-bc9287317ac3 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-c1cab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c1cab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c1cab.18b2cc729207f49f namespace: kserve-ci-e2e-test resourceVersion: "23786" uid: e631554b-b239-4ca7-a333-d7745b5108b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "23749" uid: 52544e43-805b-4ef3-906c-bc9287317ac3 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c1cab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c1cab.18b2cc7292084869 namespace: kserve-ci-e2e-test resourceVersion: "23788" uid: 8c4f471f-c76f-4aa1-9ab8-44565501f610 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24450" uid: 52544e43-805b-4ef3-906c-bc9287317ac3 kind: Event lastTimestamp: "2026-05-25T11:47:23Z" message: InferenceService [error-404-isvc-c1cab] is Ready metadata: creationTimestamp: "2026-05-25T11:47:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:23Z" name: error-404-isvc-c1cab.18b2cc7dea626c4c namespace: kserve-ci-e2e-test resourceVersion: "24451" uid: a450eab8-2075-4191-a000-4ae51730e8ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:46:05.915316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23424" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3e68-predictor-b87f65c65-xxt7d to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6bbfbf7887 namespace: kserve-ci-e2e-test resourceVersion: "23428" uid: d18a48f8-af52-4d85-8d39-3abcacb23242 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c3e68-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:06Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6bc743a927 namespace: kserve-ci-e2e-test resourceVersion: "23437" uid: 7b67f079-404b-4f22-ac0d-6abd1bc9df08 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23432" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:46:06Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6bfd5fa62d namespace: kserve-ci-e2e-test resourceVersion: "23456" uid: e143ff25-03a3-4fe4-8f24-46ff1668e0de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:06Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6bfec047d3 namespace: kserve-ci-e2e-test resourceVersion: "23461" uid: 43b7994c-0faa-4d56-8d75-8b697cae01ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6c0199671c namespace: kserve-ci-e2e-test resourceVersion: "23464" uid: d5af4d3e-c18d-4f91-9d07-f37dbd586901 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6c020dac80 namespace: kserve-ci-e2e-test resourceVersion: "23466" uid: 27c4489e-9036-487a-8c5f-901f0e51cfde reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6c0219e34f namespace: kserve-ci-e2e-test resourceVersion: "23467" uid: e5c47b42-1812-4943-a0f1-b16a547e0ac8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6c05068288 namespace: kserve-ci-e2e-test resourceVersion: "23468" uid: 35cfb8c6-c774-4005-8dae-375685ace4fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6c058df245 namespace: kserve-ci-e2e-test resourceVersion: "23469" uid: 8a2dfcb4-a077-4d3d-8710-55350c3c22c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:46:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:44Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc6c68cb35dc namespace: kserve-ci-e2e-test resourceVersion: "23922" uid: 1050f489-d368-46a4-a844-680e24a3539f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc7c28ab7642 namespace: kserve-ci-e2e-test resourceVersion: "24317" uid: 175cdfa3-ae34-4e8a-96e4-1edc98da0642 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: f1814527-8ba1-4118-ad92-4454338b4116 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d.18b2cc7c28abf75b namespace: kserve-ci-e2e-test resourceVersion: "24319" uid: ce623ce9-87a5-4f0b-8e09-a20b50f43389 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3e68-predictor-b87f65c65 namespace: kserve-ci-e2e-test resourceVersion: "23415" uid: 4fb8a2c3-6551-497a-bd5c-e8eb67c677e8 kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'Created pod: error-404-isvc-c3e68-predictor-b87f65c65-xxt7d' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-c3e68-predictor-b87f65c65.18b2cc6bbf6ad5e9 namespace: kserve-ci-e2e-test resourceVersion: "23426" uid: a5cd84da-a15b-4dbc-85df-6fd03b1f103a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23411" uid: 00c47a3d-3df3-42b2-804c-96f1f96d026a kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Scaled up replica set error-404-isvc-c3e68-predictor-b87f65c65 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-c3e68-predictor.18b2cc6bbda8c25e namespace: kserve-ci-e2e-test resourceVersion: "23418" uid: b5009d5f-5a3f-4a10-8ba2-8d579d4cc508 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23414" uid: b7675d85-1566-4aee-b574-81892b69a333 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c3e68-predictor.18b2cc6f3c51a6ac namespace: kserve-ci-e2e-test resourceVersion: "23823" uid: 1fdc49dd-ae0b-439f-92ed-fb7bc426ac2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23414" uid: b7675d85-1566-4aee-b574-81892b69a333 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:35Z" name: error-404-isvc-c3e68-predictor.18b2cc6f3ceef783 namespace: kserve-ci-e2e-test resourceVersion: "23824" uid: efae2795-1a04-4a76-9283-529dcede5fb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23589" uid: b7675d85-1566-4aee-b574-81892b69a333 kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: error-404-isvc-c3e68-predictor.18b2cc763a908ae8 namespace: kserve-ci-e2e-test resourceVersion: "24133" uid: cd1011a5-4101-47db-bbf9-03dfe3f60300 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23589" uid: b7675d85-1566-4aee-b574-81892b69a333 kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: error-404-isvc-c3e68-predictor.18b2cc763b0327a2 namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: 8110efe0-a241-4d58-ad2c-15b09a1d7afa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "23384" uid: 39adb5cf-ec99-4d93-9a6a-6826ab1aa806 kind: Event lastTimestamp: "2026-05-25T11:46:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:54Z" name: error-404-isvc-c3e68.18b2cc6bbdaa3a16 namespace: kserve-ci-e2e-test resourceVersion: "24024" uid: 88b5b2c1-3220-4b2c-b281-7c49c717ec0e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "23384" uid: 39adb5cf-ec99-4d93-9a6a-6826ab1aa806 kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3e68": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3e68": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-c3e68.18b2cc6bbe4a16e4 namespace: kserve-ci-e2e-test resourceVersion: "23421" uid: f744d0c7-5efd-433e-b40e-ed4edfa2c25d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "23384" uid: 39adb5cf-ec99-4d93-9a6a-6826ab1aa806 kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3e68": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:05Z" name: error-404-isvc-c3e68.18b2cc6bbe4a4e2e namespace: kserve-ci-e2e-test resourceVersion: "23422" uid: bf7bb692-43bb-45f6-bbf0-acd49024e067 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "24022" uid: 39adb5cf-ec99-4d93-9a6a-6826ab1aa806 kind: Event lastTimestamp: "2026-05-25T11:46:54Z" message: InferenceService [error-404-isvc-c3e68] is Ready metadata: creationTimestamp: "2026-05-25T11:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:54Z" name: error-404-isvc-c3e68.18b2cc772400ef10 namespace: kserve-ci-e2e-test resourceVersion: "24023" uid: 97a469a9-a9b7-4cf9-b50c-c6e7451c67ee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:47:16.451203Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24324" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c2c03a74f namespace: kserve-ci-e2e-test resourceVersion: "24328" uid: 7b6aacb9-0662-4129-ae49-17d6f8770210 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c7f41-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c35b85edb namespace: kserve-ci-e2e-test resourceVersion: "24341" uid: 99d32fae-ccba-4a98-820d-0b1284ecfc35 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24334" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c694d008c namespace: kserve-ci-e2e-test resourceVersion: "24361" uid: 6ef41fd6-142c-43e6-ba33-685466046b91 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c6adba631 namespace: kserve-ci-e2e-test resourceVersion: "24363" uid: 062aa145-1217-44d3-8536-9dbf667675a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c6e0ae285 namespace: kserve-ci-e2e-test resourceVersion: "24364" uid: 819872c6-7fee-4135-8993-2dde1a79f4ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c6e7dd725 namespace: kserve-ci-e2e-test resourceVersion: "24365" uid: 4f32ea07-858b-49c3-820c-a2a379736754 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c6e8ae8cd namespace: kserve-ci-e2e-test resourceVersion: "24366" uid: aa0bf4d6-adb7-4a9d-8c99-111a92c716ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c7196477f namespace: kserve-ci-e2e-test resourceVersion: "24367" uid: 850af0db-861b-456c-b41c-77b0e83598a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:17Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7c720fd173 namespace: kserve-ci-e2e-test resourceVersion: "24368" uid: add864e4-f2c5-46e6-9366-d584cb2833a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:47:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:47:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:55Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2cc7cc3f461dc namespace: kserve-ci-e2e-test resourceVersion: "24746" uid: 5f0517d1-f78d-4555-ae85-2425b30c5801 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:56:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:56:31Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2ccfd4c51684d namespace: kserve-ci-e2e-test resourceVersion: "28711" uid: 3046300a-4bdb-4fd1-bc8b-251b89a1f7b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6 namespace: kserve-ci-e2e-test resourceVersion: "24326" uid: e1ac9df7-55b7-4b46-a326-fd3d3d803fda kind: Event lastTimestamp: "2026-05-25T11:56:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:56:31Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6.18b2ccfd4c51d0d3 namespace: kserve-ci-e2e-test resourceVersion: "28712" uid: 46fc7a29-bfe9-4192-95f3-781490649af4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c7f41-predictor-78fcf75fdf namespace: kserve-ci-e2e-test resourceVersion: "24314" uid: a5c96900-1ccb-4b6e-b12a-a37426f079bd kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'Created pod: error-404-isvc-c7f41-predictor-78fcf75fdf-r2ff6' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c7f41-predictor-78fcf75fdf.18b2cc7c2b784711 namespace: kserve-ci-e2e-test resourceVersion: "24325" uid: 4a262ab8-d208-40a8-97b4-b989455c6b54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 3f3de0ed-bd1c-4338-b3e2-8456ca94a94e kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Scaled up replica set error-404-isvc-c7f41-predictor-78fcf75fdf from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c7f41-predictor.18b2cc7c28a8c719 namespace: kserve-ci-e2e-test resourceVersion: "24315" uid: 96f509d5-affb-46c2-a0c9-efb535fd2fa5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24313" uid: 63befb6c-1e6e-4a25-928d-1763c904f562 kind: Event lastTimestamp: "2026-05-25T11:47:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:46Z" name: error-404-isvc-c7f41-predictor.18b2cc7fa74ef2cd namespace: kserve-ci-e2e-test resourceVersion: "24674" uid: ae1812a3-b197-4c5b-bca0-566aab183435 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24313" uid: 63befb6c-1e6e-4a25-928d-1763c904f562 kind: Event lastTimestamp: "2026-05-25T11:47:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:46Z" name: error-404-isvc-c7f41-predictor.18b2cc7fa7bee1d2 namespace: kserve-ci-e2e-test resourceVersion: "24675" uid: ae9f20c9-037e-437e-8f56-b085aff290fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 63befb6c-1e6e-4a25-928d-1763c904f562 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:16Z" name: error-404-isvc-c7f41-predictor.18b2cc86a53fb9aa namespace: kserve-ci-e2e-test resourceVersion: "24946" uid: 0bf91aa9-d151-4bb5-be3a-6d85f4c9677b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24514" uid: 63befb6c-1e6e-4a25-928d-1763c904f562 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:16Z" name: error-404-isvc-c7f41-predictor.18b2cc86a5b54652 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: e8df3dce-24de-4ad7-a04d-4072370d6359 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24284" uid: 3d18237a-5a17-471b-abe0-7349c8619fe3 kind: Event lastTimestamp: "2026-05-25T11:48:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:48:05Z" name: error-404-isvc-c7f41.18b2cc7c289a56a5 namespace: kserve-ci-e2e-test resourceVersion: "24838" uid: 28dcb3f8-64d2-4018-a499-50b21f8bc54d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24284" uid: 3d18237a-5a17-471b-abe0-7349c8619fe3 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-c7f41": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c7f41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c7f41.18b2cc7c290e9406 namespace: kserve-ci-e2e-test resourceVersion: "24320" uid: b90ad3c4-8ccd-468e-9084-2818003e2ae9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24284" uid: 3d18237a-5a17-471b-abe0-7349c8619fe3 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c7f41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:16Z" name: error-404-isvc-c7f41.18b2cc7c290efc8a namespace: kserve-ci-e2e-test resourceVersion: "24322" uid: 53cc8c75-e4ef-4001-a9aa-9d2848eed8e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24836" uid: 3d18237a-5a17-471b-abe0-7349c8619fe3 kind: Event lastTimestamp: "2026-05-25T11:48:05Z" message: InferenceService [error-404-isvc-c7f41] is Ready metadata: creationTimestamp: "2026-05-25T11:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:48:05Z" name: error-404-isvc-c7f41.18b2cc877ed2ec39 namespace: kserve-ci-e2e-test resourceVersion: "24837" uid: 52d8dcd8-3c31-4867-be6e-9f049984c2b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:26:55.286328Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f16f5-predictor-d9979f5f9-2b89m to ip-10-0-128-146.ec2.internal metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5fd8ea5ad9 namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: d61fc7d1-e3c5-47d9-831d-b3c967ec4449 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ff2594f8b namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: dcf3a56b-272d-4268-9faf-20fdd3018d7a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ff3c58e43 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 9a5eb4e2-590f-4fc2-98c8-08e121bbbe38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ff6a7ad56 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: a7b0dda2-24d4-4316-ae7b-2633a53c0e70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ff708b2a9 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 186e17bf-9499-4a93-9c28-9e12ea451c7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ff714c316 namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: 21d0ad47-b04b-45a7-8c9b-8358bfff44b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ff9decdaf namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: 69594f7f-f043-41de-916e-4fbef618eafa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ffa44975d namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: b9794744-88ec-41fc-85f4-b60cc1697638 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cb5ffcc45cd4 namespace: kserve-ci-e2e-test resourceVersion: "18606" uid: a8b18cb7-7d63-4608-a8c9-9be9e8477380 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cbe10be0136c namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: b228cc5b-af54-4c54-90a6-8404a66ca421 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:10Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cbe10be0829d namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: 2147bd4b-4f8c-40ba-baa4-d459efc9db30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 646904be-ae32-46bb-8ccc-b3b1e359cf48 kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m.18b2cbe1711b991c namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 681acf4e-3c50-4d5f-8bd4-4f3e70110133 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-146.ec2.internal source: component: kubelet host: ip-10-0-128-146.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f16f5-predictor-d9979f5f9 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: c69295d3-76a0-40d2-864f-2f94d535aaa7 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'Created pod: error-404-isvc-f16f5-predictor-d9979f5f9-2b89m' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor-d9979f5f9.18b2cb5fd875921e namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 6d642ebb-6e77-432e-a324-911e6a5135f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: bc7e6415-0fbb-4b35-a473-0c0743cfa272 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Scaled up replica set error-404-isvc-f16f5-predictor-d9979f5f9 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5-predictor.18b2cb5fd6d91336 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 5e8b8093-14c7-4eaa-aed8-7e8b06a9cfbe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 6aef8690-53b0-4b1f-8ccf-8550ed151a5a kind: Event lastTimestamp: "2026-05-25T11:27:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:25Z" name: error-404-isvc-f16f5-predictor.18b2cb63555fdbd2 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 1e7c5d15-72cb-4892-a3ef-4ccb56244d62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 6aef8690-53b0-4b1f-8ccf-8550ed151a5a kind: Event lastTimestamp: "2026-05-25T11:27:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:25Z" name: error-404-isvc-f16f5-predictor.18b2cb6355c34e5f namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 09de22af-66d2-4f56-8e4c-3ee77115be98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 6aef8690-53b0-4b1f-8ccf-8550ed151a5a kind: Event lastTimestamp: "2026-05-25T11:28:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:10Z" name: error-404-isvc-f16f5-predictor.18b2cb6a5359169a namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: f8dedbcd-a4af-49d0-8f10-d220a4c36137 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 6aef8690-53b0-4b1f-8ccf-8550ed151a5a kind: Event lastTimestamp: "2026-05-25T11:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:10Z" name: error-404-isvc-f16f5-predictor.18b2cb6a53cc7853 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 788b5d36-7175-4a87-b3fe-9ae8700d3cc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 93e82197-03d8-4ddc-87b9-618aa4b3e0e9 kind: Event lastTimestamp: "2026-05-25T11:27:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:42Z" name: error-404-isvc-f16f5.18b2cb5fd6afb4d7 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 00515743-73f2-4fed-b20e-baec2efcd0ab reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 93e82197-03d8-4ddc-87b9-618aa4b3e0e9 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-f16f5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f16f5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5.18b2cb5fd702a696 namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: 02f37036-7a5b-4ad1-8148-36c33b2844c6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 93e82197-03d8-4ddc-87b9-618aa4b3e0e9 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f16f5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:55Z" name: error-404-isvc-f16f5.18b2cb5fd702eae6 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: e1b224c8-c959-411d-a500-666ca743ece6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: 93e82197-03d8-4ddc-87b9-618aa4b3e0e9 kind: Event lastTimestamp: "2026-05-25T11:27:41Z" message: InferenceService [error-404-isvc-f16f5] is Ready metadata: creationTimestamp: "2026-05-25T11:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:41Z" name: error-404-isvc-f16f5.18b2cb6ab7bdd050 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 3a7f0f65-5cda-4bf6-b44b-99e381643873 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:24:54.971375Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12662" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:24:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:24:54Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb43d595d5f5 namespace: kserve-ci-e2e-test resourceVersion: "12670" uid: 51dc7354-d16c-4327-94a2-939491cc4d00 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12674" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:24:55Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb43ef5ab205 namespace: kserve-ci-e2e-test resourceVersion: "12756" uid: de1e38f1-4b18-4225-83d6-512cc624105c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:320be4ef9f9314b65b9bd237a7c45b7de10514c36b5a23979ebdf05ec9d78d86" metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:55Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb43f1071961 namespace: kserve-ci-e2e-test resourceVersion: "12760" uid: f56b84ce-3402-48a2-bff4-612b9a1ef425 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:01Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:320be4ef9f9314b65b9bd237a7c45b7de10514c36b5a23979ebdf05ec9d78d86" in 5.577s (5.577s including waiting). Image size: 303117430 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:01Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb453d7c7a9e namespace: kserve-ci-e2e-test resourceVersion: "12869" uid: 9e9f8b7c-0b9e-4d79-afcd-317f37314d87 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T11:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:01Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb4541644b84 namespace: kserve-ci-e2e-test resourceVersion: "12872" uid: c030bea1-8414-4d0a-bb10-ec4cd72c3850 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T11:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:01Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb4541fce3bd namespace: kserve-ci-e2e-test resourceVersion: "12874" uid: 9fd2e4ac-9878-48be-8f0d-5850873b7013 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:06Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1534" metadata: creationTimestamp: "2026-05-25T11:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:06Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb469d642d72 namespace: kserve-ci-e2e-test resourceVersion: "12937" uid: 1cb35707-fdf3-4b27-b430-dda3191fd680 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1534" in 9.332s (9.332s including waiting). Image size: 1560931766 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb48c9a5cdf2 namespace: kserve-ci-e2e-test resourceVersion: "13033" uid: ba4a823e-1590-4af7-a5d3-14190789699a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb48ce2c75a4 namespace: kserve-ci-e2e-test resourceVersion: "13037" uid: 4958fedc-0244-44a4-8253-84ea00bc0aee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb48cec11e6e namespace: kserve-ci-e2e-test resourceVersion: "13039" uid: 96d10292-88b2-49a8-9afa-6aa63c43fca9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb48cede560e namespace: kserve-ci-e2e-test resourceVersion: "13040" uid: d338ac45-f556-4ee6-9954-ae0e5659eeb1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb48d2f6f748 namespace: kserve-ci-e2e-test resourceVersion: "13041" uid: 54b99890-718a-417c-b32f-19c0da6d1482 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb48d3799063 namespace: kserve-ci-e2e-test resourceVersion: "13042" uid: 6a83f52a-6469-425f-888a-0939eec203d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T11:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:26:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:13Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb492fa10b62 namespace: kserve-ci-e2e-test resourceVersion: "13556" uid: f42ec00e-1fe9-4c85-a3b3-dbd37f33f3a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb5fd46b7bbf namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: e9d25e41-4f1c-43d8-a584-f793b3acf2b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb5fd46cec1b namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: a367d3d3-1088-414f-9c31-48de820831de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: f83110a4-fc38-4dc8-b29a-e5010f5fa423 kind: Event lastTimestamp: "2026-05-25T11:26:58Z" message: 'Readiness probe failed: Get "https://10.133.0.16:8643/healthz": dial tcp 10.133.0.16:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:58Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt.18b2cb60b3ecc79b namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: 37999224-1347-4d56-a57b-52577afe836e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5474d9d7fb namespace: kserve-ci-e2e-test resourceVersion: "12653" uid: 7eecd376-fc2c-495a-9f9d-7d61a7913533 kind: Event lastTimestamp: "2026-05-25T11:24:54Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5474d9d7fb-c7tvt' metadata: creationTimestamp: "2026-05-25T11:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:54Z" name: isvc-sklearn-graph-1-predictor-5474d9d7fb.18b2cb43d4f1ae59 namespace: kserve-ci-e2e-test resourceVersion: "12665" uid: 7e709181-3d0e-4df4-910a-f31ca4ba8f43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12652" uid: cc9cecad-9cce-4cff-b675-c4357fafcf51 kind: Event lastTimestamp: "2026-05-25T11:24:54Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5474d9d7fb from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:54Z" name: isvc-sklearn-graph-1-predictor.18b2cb43cee3150c namespace: kserve-ci-e2e-test resourceVersion: "12654" uid: 718e92d1-bd02-4432-8645-3426d5ac9070 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:25:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12663" uid: 9546211c-4b60-4f4f-9afb-05fe0657a218 kind: Event lastTimestamp: "2026-05-25T11:25:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:25Z" name: isvc-sklearn-graph-1-predictor.18b2cb4754a4e32c namespace: kserve-ci-e2e-test resourceVersion: "13128" uid: a5d8a84c-dc7d-41a3-845f-1cf0738b3050 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:25:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12663" uid: 9546211c-4b60-4f4f-9afb-05fe0657a218 kind: Event lastTimestamp: "2026-05-25T11:25:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:25Z" name: isvc-sklearn-graph-1-predictor.18b2cb4755306075 namespace: kserve-ci-e2e-test resourceVersion: "13129" uid: 029d5388-c82e-475f-8c39-12242f760e4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12974" uid: 9546211c-4b60-4f4f-9afb-05fe0657a218 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: isvc-sklearn-graph-1-predictor.18b2cb4e53cace38 namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: eb56941e-f1ca-469a-903c-02f73104f9ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12974" uid: 9546211c-4b60-4f4f-9afb-05fe0657a218 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: isvc-sklearn-graph-1-predictor.18b2cb4e54c2acc2 namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: e30084a5-c756-4d5b-a927-a96f3983ef60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-25T11:24:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "12635" uid: 0038f886-6da6-4892-8cd3-fc5357ac38c8 kind: Event lastTimestamp: "2026-05-25T11:26:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:24Z" name: isvc-sklearn-graph-1.18b2cb43d53830e4 namespace: kserve-ci-e2e-test resourceVersion: "13674" uid: f2b494e3-4ff7-4539-b4ff-dc56c558eab2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: 0038f886-6da6-4892-8cd3-fc5357ac38c8 kind: Event lastTimestamp: "2026-05-25T11:26:24Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-25T11:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:24Z" name: isvc-sklearn-graph-1.18b2cb5892c2df75 namespace: kserve-ci-e2e-test resourceVersion: "13673" uid: 89d925b7-db8c-4d28-b8bd-e04e7539d2e6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:24:56.132624Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12792" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn to ip-10-0-131-36.ec2.internal metadata: creationTimestamp: "2026-05-25T11:24:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:24:56Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb441acd228f namespace: kserve-ci-e2e-test resourceVersion: "12796" uid: c4bc46a9-8f0d-4360-a137-2b1e78c98a62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12800" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:24:56Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:24:56Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb4433bc5fcc namespace: kserve-ci-e2e-test resourceVersion: "12808" uid: 0c09f749-1f1a-4ec2-9e96-4c28fc5a9252 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:24:56Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:320be4ef9f9314b65b9bd237a7c45b7de10514c36b5a23979ebdf05ec9d78d86" metadata: creationTimestamp: "2026-05-25T11:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:56Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb4435446721 namespace: kserve-ci-e2e-test resourceVersion: "12810" uid: e251f36a-6d41-40b9-af50-b5d21fd13407 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:24:59Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:320be4ef9f9314b65b9bd237a7c45b7de10514c36b5a23979ebdf05ec9d78d86" in 3.14s (3.14s including waiting). Image size: 303117430 bytes.' metadata: creationTimestamp: "2026-05-25T11:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:59Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb44f07016d8 namespace: kserve-ci-e2e-test resourceVersion: "12844" uid: b126519c-389a-477a-a1b1-359c18f8a5e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:24:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T11:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:59Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb44f35f35de namespace: kserve-ci-e2e-test resourceVersion: "12845" uid: c7dae3e3-3c13-4f36-b87d-b944c0fcf957 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:24:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T11:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:59Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb44f3cc4156 namespace: kserve-ci-e2e-test resourceVersion: "12846" uid: ca021059-b4a2-4de9-ab55-f4227e09052e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1534" metadata: creationTimestamp: "2026-05-25T11:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:04Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb461502dcba namespace: kserve-ci-e2e-test resourceVersion: "12912" uid: 9e87ec4c-9c16-441f-93fd-1cfd07af49e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1534" in 12.005s (12.005s including waiting). Image size: 1560931766 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb48e09c7f9d namespace: kserve-ci-e2e-test resourceVersion: "13047" uid: d25b2f55-f8dd-4110-9d13-ff268d9ed468 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb48e3c4c419 namespace: kserve-ci-e2e-test resourceVersion: "13048" uid: 5b972906-b874-4db4-a46e-34e2a7df91ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb48e433134d namespace: kserve-ci-e2e-test resourceVersion: "13049" uid: 9b7458c1-9888-4aed-a265-8b0b0ef78ae5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb48e43ee20f namespace: kserve-ci-e2e-test resourceVersion: "13050" uid: 0ec18de4-0b59-4541-8f77-15cf74055475 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:18Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.14s (2.14s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:18Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb4963d2e84d namespace: kserve-ci-e2e-test resourceVersion: "13071" uid: b90c9d60-7541-45e1-97fc-ba3f84a43f7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:18Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb4966d0cc8a namespace: kserve-ci-e2e-test resourceVersion: "13073" uid: 8e3946e6-7e8c-4ed7-a0ea-d1657e88e457 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:25:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:18Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb49674c0936 namespace: kserve-ci-e2e-test resourceVersion: "13074" uid: 2cbcff94-386e-4ceb-a3a4-143e3a55afd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T11:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb49d2479105 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 6dcdedf9-27c4-48fa-b9bd-a364284e8e90 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb5fd06ea602 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: bf34eea1-877e-4da8-afc3-924ea9daebb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb5fd06f1ce0 namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 8c93154e-5bab-4235-8bcc-e8769fadf57f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn namespace: kserve-ci-e2e-test resourceVersion: "12794" uid: 161ae216-7eee-4690-af52-9f3c3ade74ba kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn.18b2cb602c6ce3ca namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 2654c4aa-3411-42e7-894e-aed36832ff29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6465d87db8 namespace: kserve-ci-e2e-test resourceVersion: "12775" uid: 008a17db-f10a-4f12-92b2-c3ab0d142e16 kind: Event lastTimestamp: "2026-05-25T11:24:56Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6465d87db8-2q8dn' metadata: creationTimestamp: "2026-05-25T11:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:56Z" name: isvc-sklearn-graph-2-predictor-6465d87db8.18b2cb441a3b50a4 namespace: kserve-ci-e2e-test resourceVersion: "12793" uid: 1a6b2d8d-0063-43e6-ae79-27243292b94b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12773" uid: e12db912-c2b7-4a03-832a-a35613128baa kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6465d87db8 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: isvc-sklearn-graph-2-predictor.18b2cb4400fa0d4d namespace: kserve-ci-e2e-test resourceVersion: "12777" uid: 34224b4e-bbba-4754-a08f-51d10ad3a634 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12774" uid: ddacf6e3-fc19-4c0d-b5b0-d6686b2b5498 kind: Event lastTimestamp: "2026-05-25T11:25:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:55Z" name: isvc-sklearn-graph-2-predictor.18b2cb477f95466c namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: f70c1281-ea62-4ba7-91e9-d0249ea1c794 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12774" uid: ddacf6e3-fc19-4c0d-b5b0-d6686b2b5498 kind: Event lastTimestamp: "2026-05-25T11:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:55Z" name: isvc-sklearn-graph-2-predictor.18b2cb478027f868 namespace: kserve-ci-e2e-test resourceVersion: "13400" uid: ad8eff07-381b-4f12-babe-4a5f95c75c67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12994" uid: ddacf6e3-fc19-4c0d-b5b0-d6686b2b5498 kind: Event lastTimestamp: "2026-05-25T11:26:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:40Z" name: isvc-sklearn-graph-2-predictor.18b2cb4e7de7d670 namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 9899a63a-054f-40ce-89db-68f2af3fde8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12994" uid: ddacf6e3-fc19-4c0d-b5b0-d6686b2b5498 kind: Event lastTimestamp: "2026-05-25T11:26:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:40Z" name: isvc-sklearn-graph-2-predictor.18b2cb4e7e79c10b namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: a0ccbcff-01b8-40eb-9e4d-66ac78fa4f64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "12748" uid: 5416a7ea-822b-4c5c-8b14-6d3217025d05 kind: Event lastTimestamp: "2026-05-25T11:26:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:27Z" name: isvc-sklearn-graph-2.18b2cb4400cce5bc namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: 695fb726-3411-4d61-a68a-c3c43aaff1ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13861" uid: 5416a7ea-822b-4c5c-8b14-6d3217025d05 kind: Event lastTimestamp: "2026-05-25T11:26:27Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-25T11:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:27Z" name: isvc-sklearn-graph-2.18b2cb5953105739 namespace: kserve-ci-e2e-test resourceVersion: "13864" uid: 489b6f90-2b79-4ca1-9a9b-1f7075237826 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:24:55.274136Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12729" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-4wt2l to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:24:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb43e7a2fad3 namespace: kserve-ci-e2e-test resourceVersion: "12739" uid: 242e05a5-9ab1-4bad-a68a-4694990dca73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12744" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:24:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4402710ad3 namespace: kserve-ci-e2e-test resourceVersion: "12782" uid: 7430f8a4-2d48-430f-9b29-6142182ac731 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:320be4ef9f9314b65b9bd237a7c45b7de10514c36b5a23979ebdf05ec9d78d86" metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4403f3f406 namespace: kserve-ci-e2e-test resourceVersion: "12784" uid: febf4f03-bcaf-4f78-b5b0-b5867bcd2093 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:00Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:320be4ef9f9314b65b9bd237a7c45b7de10514c36b5a23979ebdf05ec9d78d86" in 5.247s (5.247s including waiting). Image size: 303117430 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb453cb7353b namespace: kserve-ci-e2e-test resourceVersion: "12868" uid: 0c4e2beb-4d68-4038-9e11-fa2872108dc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T11:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb454120d2b7 namespace: kserve-ci-e2e-test resourceVersion: "12871" uid: d44702ea-6463-43ab-9255-a61c3cfbf2f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T11:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb45419b83aa namespace: kserve-ci-e2e-test resourceVersion: "12873" uid: 51add281-9670-4d1b-a9ad-f026f6af33ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:05Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-25T11:25:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb46612255f5 namespace: kserve-ci-e2e-test resourceVersion: "12922" uid: 4abb8fe5-ef00-4e3c-91ad-46c87e218b0c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:26Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.953s (20.953s including waiting). Image size: 1306319098 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:26Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4b42124d59 namespace: kserve-ci-e2e-test resourceVersion: "13152" uid: 3bfc1b82-214f-44f5-9b21-91ae9b202943 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:26Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4b47687e71 namespace: kserve-ci-e2e-test resourceVersion: "13154" uid: bd1df6a8-903b-498d-bdd5-d038d911dc09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:26Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4b48774692 namespace: kserve-ci-e2e-test resourceVersion: "13155" uid: 08e780c2-81f4-4406-ab6f-f26051d511b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:26Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4b488f1911 namespace: kserve-ci-e2e-test resourceVersion: "13156" uid: 15a30fa5-8348-4d10-a98e-2f9de1943c63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4b4cdf4a22 namespace: kserve-ci-e2e-test resourceVersion: "13157" uid: aa9f4b1c-3143-4c33-ac86-99f000a90e29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:25:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4b4e58af68 namespace: kserve-ci-e2e-test resourceVersion: "13159" uid: a8fcf481-9508-4109-897c-c7ad739f293c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T11:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:26:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:15Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb4bc18239e6 namespace: kserve-ci-e2e-test resourceVersion: "13587" uid: 0385dbf2-4835-45e9-aebb-70062c0e010d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb5fcd0d37d2 namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: e2a98669-03b6-45b7-86eb-5d539adfdff1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l namespace: kserve-ci-e2e-test resourceVersion: "12734" uid: 6bf106cb-fa28-4201-9a8e-1f1a8a6fa055 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-4wt2l.18b2cb5fcd0dbada namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 86e5c7d8-9957-4820-90e5-35fb1e3964fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "12708" uid: 3c416063-b814-4b82-9b85-16c01b8354c5 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-4wt2l' metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b2cb43e6d98e0f namespace: kserve-ci-e2e-test resourceVersion: "12732" uid: 58a17675-b4f8-475e-9755-d3558bb5fced reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12702" uid: 5acce099-b948-4a34-bc4a-6ecbabacfe6f kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: isvc-xgboost-graph-predictor.18b2cb43e09318e4 namespace: kserve-ci-e2e-test resourceVersion: "12710" uid: ecdeab3c-b19f-4d96-afb1-e9d492cb3af0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12706" uid: 550673e7-3246-4d0a-bb31-478868f1b232 kind: Event lastTimestamp: "2026-05-25T11:26:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:10Z" name: isvc-xgboost-graph-predictor.18b2cb475f32d762 namespace: kserve-ci-e2e-test resourceVersion: "13526" uid: 7b157ffa-6dbc-4879-a6e9-db0510a67449 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12706" uid: 550673e7-3246-4d0a-bb31-478868f1b232 kind: Event lastTimestamp: "2026-05-25T11:26:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:10Z" name: isvc-xgboost-graph-predictor.18b2cb475fc40ed7 namespace: kserve-ci-e2e-test resourceVersion: "13527" uid: 323d27a3-9bd0-456d-a677-680af040b9c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12981" uid: 550673e7-3246-4d0a-bb31-478868f1b232 kind: Event lastTimestamp: "2026-05-25T11:26:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:40Z" name: isvc-xgboost-graph-predictor.18b2cb58d9b7914c namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 57126770-6be4-4b43-bfec-17778bb07a9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12981" uid: 550673e7-3246-4d0a-bb31-478868f1b232 kind: Event lastTimestamp: "2026-05-25T11:26:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:40Z" name: isvc-xgboost-graph-predictor.18b2cb58da229d53 namespace: kserve-ci-e2e-test resourceVersion: "13971" uid: c7d633a5-e95a-445e-89db-5494f6aa4411 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "12695" uid: 314721cb-6172-4950-af15-25ab1f9ca1f0 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:25Z" name: isvc-xgboost-graph.18b2cb43e068be23 namespace: kserve-ci-e2e-test resourceVersion: "13713" uid: 5401ff08-d05a-49d3-994f-31d2d901c2e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13696" uid: 314721cb-6172-4950-af15-25ab1f9ca1f0 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:25Z" name: isvc-xgboost-graph.18b2cb58d08278a1 namespace: kserve-ci-e2e-test resourceVersion: "13698" uid: 7bf5720e-0a79-4cce-adf3-9318a628d585 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:26:44.910336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14028" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7d974fc4cf-t4hfw to ip-10-0-131-36.ec2.internal metadata: creationTimestamp: "2026-05-25T11:26:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:26:44Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5d6e7547e0 namespace: kserve-ci-e2e-test resourceVersion: "14034" uid: 43657e0b-6531-4966-94bd-b5df5b7e9c9c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:45Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:26:45Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:26:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:26:45Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5d87b47db8 namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: fc25ae7f-163c-43ca-b2cf-a8d305eda992 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:26:45Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" metadata: creationTimestamp: "2026-05-25T11:26:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:45Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5d88f2645c namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 8d943c6f-4386-4cf0-b85e-802bf8cb4716 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:26:47Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" in 1.746s (1.746s including waiting). Image size: 218005276 bytes.' metadata: creationTimestamp: "2026-05-25T11:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:47Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5df1089c84 namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: 185d4398-193d-4f9f-a19c-e6fc17ee8a40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:26:47Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-25T11:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:47Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5df3e3c94a namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: 0015c879-8d4e-4595-9fff-9a65a9ed42ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:26:47Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-25T11:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:47Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5df44fb62c namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: 821fa2c4-8233-49e8-a48c-691121c56822 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb5fc8a01a96 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 8b9c9718-c097-420a-adc6-835342f23815 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d974fc4cf-t4hfw namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 84fca304-5af8-4d5a-b8b8-9e4a4d40332d kind: Event lastTimestamp: "2026-05-25T11:27:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:23Z" name: model-chainer-7d974fc4cf-t4hfw.18b2cb60b3831036 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 94d2e979-975f-4d53-a5e3-d393a1947451 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7d974fc4cf namespace: kserve-ci-e2e-test resourceVersion: "14025" uid: 4d2fffc6-e70f-499a-a9c9-31eaa3348b97 kind: Event lastTimestamp: "2026-05-25T11:26:44Z" message: 'Created pod: model-chainer-7d974fc4cf-t4hfw' metadata: creationTimestamp: "2026-05-25T11:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:44Z" name: model-chainer-7d974fc4cf.18b2cb5d6e0f094d namespace: kserve-ci-e2e-test resourceVersion: "14030" uid: 0c65dc3d-fc12-4043-8d31-52bff8dedd14 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14023" uid: 37a39c0e-f526-45ae-9a09-60e7db16161e kind: Event lastTimestamp: "2026-05-25T11:26:44Z" message: Scaled up replica set model-chainer-7d974fc4cf from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:44Z" name: model-chainer.18b2cb5d6d685d35 namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: 5d94e446-f678-441a-8dd5-3fdf670113f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: f5fbb551-71bc-412d-94e1-7bd8c2b5798b kind: Event lastTimestamp: "2026-05-25T11:26:44Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-25T11:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:44Z" name: model-chainer.18b2cb5d72aa5d46 namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: c139f8c7-4e7e-480b-ba90-0cd8edb7a375 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:37:10.103519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-412bf-5fd95869d7-vz7sb to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbeefee02d25 namespace: kserve-ci-e2e-test resourceVersion: "19317" uid: 3557ca0c-fb1d-48f6-8a9e-edc86387510d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:10Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-412bf-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbef07acd6a4 namespace: kserve-ci-e2e-test resourceVersion: "19329" uid: fc5a2123-9fd5-4613-9557-ded9e8d41195 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:11Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:37:11Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbef3beadda3 namespace: kserve-ci-e2e-test resourceVersion: "19344" uid: 72de411c-5ea4-420f-8214-2d8db5fee3d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-412bf} kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:11Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbef3d13b0d2 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 221c577c-7b96-4bae-88d4-6788fe977079 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-412bf} kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:11Z" message: 'Created container: sequence-graph-412bf' metadata: creationTimestamp: "2026-05-25T11:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:11Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbef4000d07e namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: 520f705f-820c-466f-888a-1f93d980b914 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-412bf} kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:11Z" message: Started container sequence-graph-412bf metadata: creationTimestamp: "2026-05-25T11:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:11Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbef40742445 namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: ece7e191-f7c3-4470-b1f8-57623273c4e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-412bf} kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Stopping container sequence-graph-412bf metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbf156efc159 namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: b1abdc8c-31a3-4beb-a682-b1acff84b91d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-412bf} kind: Pod name: sequence-graph-412bf-5fd95869d7-vz7sb namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 524390a7-82fd-4c7e-991d-6298fbe2b538 kind: Event lastTimestamp: "2026-05-25T11:37:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:47Z" name: sequence-graph-412bf-5fd95869d7-vz7sb.18b2cbf1fbe94296 namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: b31650fa-f90d-4f74-9ed9-79192f68b894 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-412bf-5fd95869d7 namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 29e36b29-7b6e-4b11-bb75-09adcb163217 kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'Created pod: sequence-graph-412bf-5fd95869d7-vz7sb' metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf-5fd95869d7.18b2cbeefe7d1e1e namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: feb98aba-91c7-4d35-8b9b-7bb05510fa6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-412bf namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 9f1a8e18-9f90-4920-aba3-0f443b2ff0fd kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: Scaled up replica set sequence-graph-412bf-5fd95869d7 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf.18b2cbeefdd01940 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: c55be106-5bd0-4eb1-87bc-83f6135a8d06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-412bf namespace: kserve-ci-e2e-test resourceVersion: "19325" uid: 2909aeef-e828-4328-9330-4963a0d0d4a3 kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: InferenceGraph [sequence-graph-412bf] is Ready metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf.18b2cbef0515cfd4 namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: 7babd9e8-221e-46dd-ab79-c3a0aa32d62b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-412bf namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 2909aeef-e828-4328-9330-4963a0d0d4a3 kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'Failed to update status for InferenceGraph "sequence-graph-412bf": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-412bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf.18b2cbef07816141 namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: bc4b55e0-7ba2-4059-beaa-049b5523d15c reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-412bf namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 2909aeef-e828-4328-9330-4963a0d0d4a3 kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-412bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:10Z" name: sequence-graph-412bf.18b2cbef0781aec6 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: debfebb2-307b-4f89-bc84-aed97bc0a695 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:38:20.374727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b59e4-5f69967656-8n86s to ip-10-0-131-36.ec2.internal metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cbff5b5db84a namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: ca9d0128-2508-4cad-924d-83e186a976cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cbff74780b62 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: a91f0b2e-cdd4-46ce-b9a5-b83893643ed9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b59e4} kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cbff75db350d namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: 37ca3967-fa2d-4b48-9b76-375b27068e1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b59e4} kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: 'Created container: sequence-graph-b59e4' metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cbff78f83b76 namespace: kserve-ci-e2e-test resourceVersion: "20153" uid: 7164f183-e885-49db-b464-9233f2d1937d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b59e4} kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: Started container sequence-graph-b59e4 metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cbff7962d4a9 namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: a27ad968-54ee-4e5d-b1db-96b76fffa947 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b59e4} kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: "2026-05-25T11:46:34Z" message: Stopping container sequence-graph-b59e4 metadata: creationTimestamp: "2026-05-25T11:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:34Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cc7283857db1 namespace: kserve-ci-e2e-test resourceVersion: "23711" uid: 633946ca-8127-45aa-bb59-1763ba919a60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b59e4} kind: Pod name: sequence-graph-b59e4-5f69967656-8n86s namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: b45f18f5-07f7-48cc-9003-e72ae8602b1c kind: Event lastTimestamp: "2026-05-25T11:47:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:01Z" name: sequence-graph-b59e4-5f69967656-8n86s.18b2cc72fb945cd6 namespace: kserve-ci-e2e-test resourceVersion: "24073" uid: 89dd7a01-4b01-494b-9453-cdabbd85242f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b59e4-5f69967656 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 35a62f5a-2bbd-4cd3-a1c3-0ee2b592395f kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: 'Created pod: sequence-graph-b59e4-5f69967656-8n86s' metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4-5f69967656.18b2cbff5add706a namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 5ef44ed4-94ac-41ec-8411-588ffc571787 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: f6fa6d11-71b5-4b4a-9775-fc34f48af785 kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: Scaled up replica set sequence-graph-b59e4-5f69967656 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4.18b2cbff5a262d4b namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 0ba1ac20-2c71-4975-ae5c-2b666912f8f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: ca286b15-ba00-4577-a7b1-52d732135f54 kind: Event lastTimestamp: "2026-05-25T11:38:20Z" message: InferenceGraph [sequence-graph-b59e4] is Ready metadata: creationTimestamp: "2026-05-25T11:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:38:20Z" name: sequence-graph-b59e4.18b2cbff60b2d7ee namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 4f27e9c6-0dca-49c1-9a75-9618eb597bdf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 4f79e9b3-2204-4269-a4b8-d16deb8e5f2c kind: Event lastTimestamp: "2026-05-25T11:38:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:35Z" name: sequence-graph-b59e4.18b2cc02d8905d2e namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: 44168d0f-8c5e-401c-bc59-3861aefc77af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 4f79e9b3-2204-4269-a4b8-d16deb8e5f2c kind: Event lastTimestamp: "2026-05-25T11:38:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:35Z" name: sequence-graph-b59e4.18b2cc02d8eecf77 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: ce6b32f8-ceaf-49e4-8fbb-64a7c4b910ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 4f79e9b3-2204-4269-a4b8-d16deb8e5f2c kind: Event lastTimestamp: "2026-05-25T11:38:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:50Z" name: sequence-graph-b59e4.18b2cc06579ea8b6 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 71e9b3b5-53e2-4c91-8c36-8b939101ed1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 4f79e9b3-2204-4269-a4b8-d16deb8e5f2c kind: Event lastTimestamp: "2026-05-25T11:38:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:50Z" name: sequence-graph-b59e4.18b2cc06581525ff namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 665e1f2c-4f91-463c-9058-c971e3e1f569 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:27:55.203404Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f16f5-5b574b565f-qfm26 to ip-10-0-131-36.ec2.internal metadata: creationTimestamp: "2026-05-25T11:27:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:27:55Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cb6dcc406e34 namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 1d3d8c40-587f-4724-b335-e8f6bb227974 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:27:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f16f5-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:55Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cb6dd6b33901 namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: e0dfdcb5-89ce-4cf4-8213-89a8460d86c2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:27:56Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:27:56Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cb6e097aaf8b namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 8e49f186-72d1-4134-a102-2e88464bfc32 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f16f5} kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:27:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:56Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cb6e0ae66f24 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 85b098df-ee33-4c0d-962f-7f7c34e33cfb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f16f5} kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:27:56Z" message: 'Created container: sequence-graph-f16f5' metadata: creationTimestamp: "2026-05-25T11:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:56Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cb6e0e0cfee6 namespace: kserve-ci-e2e-test resourceVersion: "14905" uid: 7de12298-e45b-4af7-b451-491808f032e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f16f5} kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:27:56Z" message: Started container sequence-graph-f16f5 metadata: creationTimestamp: "2026-05-25T11:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:56Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cb6e0e7a0c88 namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 7f81571b-b8ad-422d-8e24-02f8157ef978 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f16f5} kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:36:09Z" message: Stopping container sequence-graph-f16f5 metadata: creationTimestamp: "2026-05-25T11:36:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:09Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cbe0fab47856 namespace: kserve-ci-e2e-test resourceVersion: "18464" uid: 5dc323c7-7ba3-4186-b6fe-525768eda9ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:36:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f16f5} kind: Pod name: sequence-graph-f16f5-5b574b565f-qfm26 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 21bad3a0-fb8a-4668-8a3b-75841ab8290a kind: Event lastTimestamp: "2026-05-25T11:36:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:38Z" name: sequence-graph-f16f5-5b574b565f-qfm26.18b2cbe1bdc2fa8a namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: 2559ad51-4dbb-4746-8ee0-553c5255a93d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f16f5-5b574b565f namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: acc93277-e5dd-4d0b-97b2-015db932467e kind: Event lastTimestamp: "2026-05-25T11:27:55Z" message: 'Created pod: sequence-graph-f16f5-5b574b565f-qfm26' metadata: creationTimestamp: "2026-05-25T11:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:55Z" name: sequence-graph-f16f5-5b574b565f.18b2cb6dcbae3752 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: c5e9ef98-e891-4707-bc37-d6cd890d01ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 1fff6610-6275-4438-a081-bc793792ba77 kind: Event lastTimestamp: "2026-05-25T11:27:55Z" message: Scaled up replica set sequence-graph-f16f5-5b574b565f from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:55Z" name: sequence-graph-f16f5.18b2cb6dcb011013 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: 1915b0db-4308-4ba3-8463-70255f80488a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: ea2e54af-573a-4e8d-b3a7-42e847a1c3e1 kind: Event lastTimestamp: "2026-05-25T11:27:55Z" message: InferenceGraph [sequence-graph-f16f5] is Ready metadata: creationTimestamp: "2026-05-25T11:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:55Z" name: sequence-graph-f16f5.18b2cb6dd0c3bbda namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 3ebfbf0f-a7e7-4b38-b674-fe636fbca5d6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:28:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: cf55394d-44b6-4244-9917-7affde8e7440 kind: Event lastTimestamp: "2026-05-25T11:28:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:10Z" name: sequence-graph-f16f5.18b2cb7149c91fde namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 4212e956-fdc1-43cd-9920-22509b6b71d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:28:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: cf55394d-44b6-4244-9917-7affde8e7440 kind: Event lastTimestamp: "2026-05-25T11:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:10Z" name: sequence-graph-f16f5.18b2cb714a15e33d namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: a778fd8e-e2ca-44c2-a10e-981d41167e80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: cf55394d-44b6-4244-9917-7affde8e7440 kind: Event lastTimestamp: "2026-05-25T11:28:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:25Z" name: sequence-graph-f16f5.18b2cb74c96c2a35 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 2c1c0b3c-aa39-49eb-9a18-675cef1cbffa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: cf55394d-44b6-4244-9917-7affde8e7440 kind: Event lastTimestamp: "2026-05-25T11:28:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:25Z" name: sequence-graph-f16f5.18b2cb74c9d90b50 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: f06a6341-155e-421f-8c3b-def4af0d7f83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:47:05.947160Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-c3e68-7bd56c4bbb-xdqlg to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:47:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:47:05Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc79b9eca6d8 namespace: kserve-ci-e2e-test resourceVersion: "24132" uid: 54daaf58-28e8-4e96-84b7-9ee730d9141f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:06Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24134" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: "2026-05-25T11:47:06Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:47:06Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc79d3017cba namespace: kserve-ci-e2e-test resourceVersion: "24147" uid: ebc95277-fd68-40cb-b892-b303ed6ae08b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c3e68} kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: "2026-05-25T11:47:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:06Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc79d44c710c namespace: kserve-ci-e2e-test resourceVersion: "24149" uid: d6ecff19-782a-4bc6-996a-94f8dab4c9a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c3e68} kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: "2026-05-25T11:47:06Z" message: 'Created container: splitter-graph-c3e68' metadata: creationTimestamp: "2026-05-25T11:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:06Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc79d6fe32b0 namespace: kserve-ci-e2e-test resourceVersion: "24150" uid: 24a4679a-637d-4f8d-890e-bad124e43e88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c3e68} kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: "2026-05-25T11:47:06Z" message: Started container splitter-graph-c3e68 metadata: creationTimestamp: "2026-05-25T11:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:06Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc79d77237cc namespace: kserve-ci-e2e-test resourceVersion: "24151" uid: a02aac52-55d4-41bc-a5f1-13ccd8e897b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c3e68} kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Stopping container splitter-graph-c3e68 metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc7c127b8dda namespace: kserve-ci-e2e-test resourceVersion: "24246" uid: 5bda0f45-a6b1-4ebe-af5a-97856cc085b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c3e68} kind: Pod name: splitter-graph-c3e68-7bd56c4bbb-xdqlg namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8bed892d-7b2a-4cf9-bb6d-3933ebe5a850 kind: Event lastTimestamp: "2026-05-25T11:47:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:42Z" name: splitter-graph-c3e68-7bd56c4bbb-xdqlg.18b2cc7c73e185c2 namespace: kserve-ci-e2e-test resourceVersion: "24645" uid: 1f7b3112-b7c2-4036-a886-0133e7ef9282 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-c3e68-7bd56c4bbb namespace: kserve-ci-e2e-test resourceVersion: "24119" uid: b81c5c66-391c-433a-a088-7a2fe385a6e2 kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'Created pod: splitter-graph-c3e68-7bd56c4bbb-xdqlg' metadata: creationTimestamp: "2026-05-25T11:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: splitter-graph-c3e68-7bd56c4bbb.18b2cc79b84d8744 namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: fdc64fb0-2a14-4db0-b5c0-255887b12e08 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "24117" uid: 05190d1a-a0f7-4668-819d-174ab6d4d628 kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: Scaled up replica set splitter-graph-c3e68-7bd56c4bbb from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: splitter-graph-c3e68.18b2cc79b5f8e477 namespace: kserve-ci-e2e-test resourceVersion: "24122" uid: a989e535-ffaf-46d3-972d-5fcae48929a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "24137" uid: 2f54e28f-5aad-4479-8110-dc9a9bed2147 kind: Event lastTimestamp: "2026-05-25T11:47:06Z" message: InferenceGraph [splitter-graph-c3e68] is Ready metadata: creationTimestamp: "2026-05-25T11:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:06Z" name: splitter-graph-c3e68.18b2cc79bd358ff4 namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: 06336c8d-cf47-4be2-b88d-d99824854f12 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:48:16.254657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24929" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-c7f41-7c9cdb5df5-28c54 to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:48:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:48:16Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2cc8a1893d3c2 namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 94dca051-25d9-415e-851f-6647ea970ea0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:16Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-c7f41-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:48:16Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2cc8a2511f331 namespace: kserve-ci-e2e-test resourceVersion: "24948" uid: 140f5e50-3988-4fd4-91c6-2aef2a5fc205 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:17Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:48:17Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:48:17Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2cc8a559c5c1d namespace: kserve-ci-e2e-test resourceVersion: "24957" uid: 95dd73d2-1ba5-4834-8c78-b5abfc7fb2b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c7f41} kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:48:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:48:17Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2cc8a56f6bce1 namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 04bf7f3c-b2a7-4bb4-9da7-888b32ca050f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c7f41} kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:48:17Z" message: 'Created container: splitter-graph-c7f41' metadata: creationTimestamp: "2026-05-25T11:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:48:17Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2cc8a59cd0aeb namespace: kserve-ci-e2e-test resourceVersion: "24961" uid: fefc0541-ab43-476d-acdc-6caf9ed45c50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c7f41} kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:48:17Z" message: Started container splitter-graph-c7f41 metadata: creationTimestamp: "2026-05-25T11:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:48:17Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2cc8a5a3cd884 namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: 080d84ae-6ca9-496e-93a1-c15da70dd92c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:56:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c7f41} kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:56:30Z" message: Stopping container splitter-graph-c7f41 metadata: creationTimestamp: "2026-05-25T11:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:56:30Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2ccfd44799074 namespace: kserve-ci-e2e-test resourceVersion: "28682" uid: 6c726f40-901f-4739-b933-a3f09e5a523b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c7f41} kind: Pod name: splitter-graph-c7f41-7c9cdb5df5-28c54 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 41ab2c3d-0282-44d9-bbdd-c2f34184c28c kind: Event lastTimestamp: "2026-05-25T11:56:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:56:58Z" name: splitter-graph-c7f41-7c9cdb5df5-28c54.18b2ccfdf3aa0d7b namespace: kserve-ci-e2e-test resourceVersion: "28913" uid: e8f5e695-87fc-4f25-b7b0-1677b5e97365 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-c7f41-7c9cdb5df5 namespace: kserve-ci-e2e-test resourceVersion: "24926" uid: 09d2650c-5157-4074-935b-f14cba83e821 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: 'Created pod: splitter-graph-c7f41-7c9cdb5df5-28c54' metadata: creationTimestamp: "2026-05-25T11:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:16Z" name: splitter-graph-c7f41-7c9cdb5df5.18b2cc8a183670dc namespace: kserve-ci-e2e-test resourceVersion: "24931" uid: 85c579c5-2892-43c1-9d87-4055d57de0be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24924" uid: ec5c8f4e-5f92-486f-b04c-40bda3c2fcc0 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: Scaled up replica set splitter-graph-c7f41-7c9cdb5df5 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:16Z" name: splitter-graph-c7f41.18b2cc8a17380d10 namespace: kserve-ci-e2e-test resourceVersion: "24928" uid: b63f4756-57c6-434c-8800-0b5d8ac542fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 9fe62efe-15d4-4a0e-b464-43b5bd979bf2 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: InferenceGraph [splitter-graph-c7f41] is Ready metadata: creationTimestamp: "2026-05-25T11:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:48:16Z" name: splitter-graph-c7f41.18b2cc8a1d7ddce7 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: fb521cdb-8829-45b1-8886-df754fad4b8a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:48:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: a1c381f3-a4fb-4216-9dce-03a52de0577f kind: Event lastTimestamp: "2026-05-25T11:48:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:46Z" name: splitter-graph-c7f41.18b2cc8d96289f8d namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: eca05176-256b-41b3-a420-17a4052c4a33 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:48:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: a1c381f3-a4fb-4216-9dce-03a52de0577f kind: Event lastTimestamp: "2026-05-25T11:48:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:46Z" name: splitter-graph-c7f41.18b2cc8d96f0bb26 namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: 20bbe17c-705f-47af-8781-541fabde068b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:36:50.846988Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19035" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-20622-predictor-68cdb5545b-mbbc8 to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbea8318d6cf namespace: kserve-ci-e2e-test resourceVersion: "19041" uid: df8d6cc2-ed8a-44fa-bce4-e969265e218d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbea9c5ec1d2 namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: 5a04f52a-8f47-48f2-9c70-450de638de2b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbea9da809c9 namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 485440ea-8dac-4b79-b8f0-b831c9753138 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbeaa04d6913 namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 6ddcf9b4-970f-44dd-aa4c-fcee58ea9715 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbeaa0c6d121 namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: ae2cf885-8973-4c04-806f-33f0c15fb6b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbeaa0d2741c namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 81bab53e-a0ba-459e-abfa-d32436c92c8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbeaa39f4a10 namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: 21efb229-9b2c-4420-a61e-c1aa9f3e655f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:36:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:51Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbeaa458b82b namespace: kserve-ci-e2e-test resourceVersion: "19105" uid: e389f2f3-6ee4-4fc5-a03b-f34d4c649f88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:37:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:28Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cbeafc04f9aa namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: a72a1e65-60e8-426e-9f9d-df0aee02eca2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cc6bb6192b40 namespace: kserve-ci-e2e-test resourceVersion: "23373" uid: feb20f0b-68eb-47fd-91db-c1c5ef3dab1f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 379c8511-fc5f-4c2a-bc27-8f6370f73c3b kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-20622-predictor-68cdb5545b-mbbc8.18b2cc6bb6199880 namespace: kserve-ci-e2e-test resourceVersion: "23376" uid: 87281918-0be5-4c04-968f-da613aa168d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-20622-predictor-68cdb5545b namespace: kserve-ci-e2e-test resourceVersion: "19025" uid: 1fdb465e-022f-4819-92de-6519a466a4c5 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: 'Created pod: success-200-isvc-20622-predictor-68cdb5545b-mbbc8' metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-20622-predictor-68cdb5545b.18b2cbea829f22af namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 49fc437a-77f0-4020-b828-118ae3773d1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19021" uid: 890ffaa2-c45d-4831-a321-f9b9935a5e18 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Scaled up replica set success-200-isvc-20622-predictor-68cdb5545b from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-20622-predictor.18b2cbea7fe2f3d4 namespace: kserve-ci-e2e-test resourceVersion: "19029" uid: 7afa11e1-4e46-4a98-be09-9147a6c08ae0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 0c713d91-75d4-47ea-a2e4-744a14a00fde kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-20622-predictor.18b2cbedfe6c6fee namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: 58f6fc05-566f-4de6-9528-73a237bf9426 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 0c713d91-75d4-47ea-a2e4-744a14a00fde kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-20622-predictor.18b2cbedfee20808 namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: 8ac7e5c9-d452-41ce-b36a-a286765c69e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: 0c713d91-75d4-47ea-a2e4-744a14a00fde kind: Event lastTimestamp: "2026-05-25T11:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:05Z" name: success-200-isvc-20622-predictor.18b2cbf4fc8322e5 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: b9ccda8d-ee63-412f-8ea3-3dd0e126974a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20622-predictor namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: 0c713d91-75d4-47ea-a2e4-744a14a00fde kind: Event lastTimestamp: "2026-05-25T11:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:05Z" name: success-200-isvc-20622-predictor.18b2cbf4fcf49706 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: 698c037a-3ef7-42d2-bbfe-32040f69e853 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19002" uid: f43a471c-156d-48a9-a1bb-0e2a42734e15 kind: Event lastTimestamp: "2026-05-25T11:37:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:39Z" name: success-200-isvc-20622.18b2cbea7fb85196 namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: d0780f84-5f45-4c6d-9dd2-768465124db6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19002" uid: f43a471c-156d-48a9-a1bb-0e2a42734e15 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-20622": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-20622": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-20622.18b2cbea802a0ba4 namespace: kserve-ci-e2e-test resourceVersion: "19030" uid: 4bf68d2c-00fb-4619-b649-8e36bb86ce03 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19002" uid: f43a471c-156d-48a9-a1bb-0e2a42734e15 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-20622": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-20622.18b2cbea802a65f3 namespace: kserve-ci-e2e-test resourceVersion: "19032" uid: 1b31fe6c-bf2b-4811-9c00-b094e5e1428f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-20622 namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: f43a471c-156d-48a9-a1bb-0e2a42734e15 kind: Event lastTimestamp: "2026-05-25T11:37:38Z" message: InferenceService [success-200-isvc-20622] is Ready metadata: creationTimestamp: "2026-05-25T11:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:38Z" name: success-200-isvc-20622.18b2cbf5b7a173fe namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 48fe5d30-900a-464e-8219-62730d0ab6f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:24:55.153318Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12703" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-21751-predictor-5c775cdf48-28pfz to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:24:55Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb43e06e0d86 namespace: kserve-ci-e2e-test resourceVersion: "12709" uid: c2676055-bf09-4e59-988c-3622656c02ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12717" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:24:55Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb43f9ccf288 namespace: kserve-ci-e2e-test resourceVersion: "12763" uid: 605abf94-d213-4ff5-860f-5e76087c0081 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:24:55Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb43fb4609d2 namespace: kserve-ci-e2e-test resourceVersion: "12765" uid: 790d46d9-f2b7-44a6-b05e-1d6c9fff84ea reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:08Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" in 12.606s (12.606s including waiting). Image size: 1335713900 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:08Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb46eaa713ba namespace: kserve-ci-e2e-test resourceVersion: "12953" uid: 49600b88-9c7b-481d-a947-6e835b03451d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:08Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb46ee94ad15 namespace: kserve-ci-e2e-test resourceVersion: "12955" uid: d23a81d6-d347-43ef-bb8f-3ceba7fe8675 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:08Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb46ef3e1b61 namespace: kserve-ci-e2e-test resourceVersion: "12956" uid: f8685d65-5fd9-43a2-8295-2f0687b2d707 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T11:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:08Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb46ef5273c2 namespace: kserve-ci-e2e-test resourceVersion: "12957" uid: 07ff700b-f5b9-48ef-834e-86254d531989 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 7.956s (7.956s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb48c98e7cab namespace: kserve-ci-e2e-test resourceVersion: "13032" uid: 164ebced-325d-4a92-91b8-ea1cf2ad413d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb48ce07b2b2 namespace: kserve-ci-e2e-test resourceVersion: "13036" uid: fc4111dc-cc19-45ae-894f-c84e37c58ab9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:16Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb48ce8f27b1 namespace: kserve-ci-e2e-test resourceVersion: "13038" uid: f0d4c550-8c56-4a3c-a29c-404749fd3eef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:25:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:25:53Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb492fa2330c namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 3b348d5a-d73a-4c17-9dba-061c25c48c21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb58daaf27a6 namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: c9e8ab6e-8964-4498-a9fe-e4763dfe2caa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21751-predictor-5c775cdf48-28pfz namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 55b6e231-c24f-430e-b952-b778e29d1159 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-21751-predictor-5c775cdf48-28pfz.18b2cb58daaf9c56 namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: 8f0ab7cd-c3d5-4ed1-8c82-129c9ad72e09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-21751-predictor-5c775cdf48 namespace: kserve-ci-e2e-test resourceVersion: "12687" uid: 01b3b1d2-6bc7-4635-8f64-c187599c0bb8 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: 'Created pod: success-200-isvc-21751-predictor-5c775cdf48-28pfz' metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: success-200-isvc-21751-predictor-5c775cdf48.18b2cb43dfe2ecd9 namespace: kserve-ci-e2e-test resourceVersion: "12704" uid: 79aac8ea-9b42-4ffd-85f3-b9316f58c0dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12685" uid: 0370ef11-6261-4cfe-a290-35b6c2bb0067 kind: Event lastTimestamp: "2026-05-25T11:24:55Z" message: Scaled up replica set success-200-isvc-21751-predictor-5c775cdf48 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:24:55Z" name: success-200-isvc-21751-predictor.18b2cb43db50cb0b namespace: kserve-ci-e2e-test resourceVersion: "12691" uid: d8838a3d-f1aa-4203-b14a-0daa516b787e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12686" uid: 2c1d385b-9dd5-4129-94aa-9fa75c1a6d83 kind: Event lastTimestamp: "2026-05-25T11:25:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:55Z" name: success-200-isvc-21751-predictor.18b2cb475a21e39e namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: 708c2af2-2a3c-4f76-bfd5-030b85f7c55d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12686" uid: 2c1d385b-9dd5-4129-94aa-9fa75c1a6d83 kind: Event lastTimestamp: "2026-05-25T11:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:25:55Z" name: success-200-isvc-21751-predictor.18b2cb475b15ea6c namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: 16f9371e-b1ba-4b05-810b-0b5a1dccb328 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12977" uid: 2c1d385b-9dd5-4129-94aa-9fa75c1a6d83 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-21751-predictor.18b2cb4e58f01f64 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: a995f9e8-59da-4af5-9677-705be88f6cee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21751-predictor namespace: kserve-ci-e2e-test resourceVersion: "12977" uid: 2c1d385b-9dd5-4129-94aa-9fa75c1a6d83 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-21751-predictor.18b2cb4e596db021 namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 3d980d47-94d0-472c-ab04-54a107fb89f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21751 namespace: kserve-ci-e2e-test resourceVersion: "12675" uid: 59a68788-d440-4ade-942a-394298eb799f kind: Event lastTimestamp: "2026-05-25T11:26:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:04Z" name: success-200-isvc-21751.18b2cb43db22ce40 namespace: kserve-ci-e2e-test resourceVersion: "13478" uid: ddaf8f3d-0aa8-4a3a-83a4-dac023a58fd2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21751 namespace: kserve-ci-e2e-test resourceVersion: "13474" uid: 59a68788-d440-4ade-942a-394298eb799f kind: Event lastTimestamp: "2026-05-25T11:26:04Z" message: InferenceService [success-200-isvc-21751] is Ready metadata: creationTimestamp: "2026-05-25T11:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:04Z" name: success-200-isvc-21751.18b2cb53edc217f5 namespace: kserve-ci-e2e-test resourceVersion: "13475" uid: ff0171ad-aa0c-4b66-889b-00f2ff1a6298 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:36:10.086673Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18499" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-412bf-predictor-556fc6f8-2s6tc to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe10597c001 namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 1671f6cf-c498-479d-a9fa-4090fa537194 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-412bf-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe11239dbc6 namespace: kserve-ci-e2e-test resourceVersion: "18554" uid: a07715b1-7919-4554-a6c2-cdbaa84998cb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18511" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe1425eb9c3 namespace: kserve-ci-e2e-test resourceVersion: "18578" uid: 1ac38a63-4ca7-4579-9ea9-899a1c229330 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe1449a8a52 namespace: kserve-ci-e2e-test resourceVersion: "18580" uid: 1fc9deb1-dfc0-4491-8104-a1af2a57a68a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe147483761 namespace: kserve-ci-e2e-test resourceVersion: "18582" uid: 4869a13a-f73e-4adf-a714-c5ee375b66b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe147b8f2a0 namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: 0930c822-0582-4063-8514-7840df2e59ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe147c4ac9a namespace: kserve-ci-e2e-test resourceVersion: "18584" uid: 619e8c60-13a6-4876-833f-267d2ad1c722 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe14a82c031 namespace: kserve-ci-e2e-test resourceVersion: "18585" uid: c9a30864-38cf-4abc-abe8-c713a362f022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:11Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe14af2e17d namespace: kserve-ci-e2e-test resourceVersion: "18586" uid: 8c94bce5-ec3c-4647-b3e1-68185f3eb461 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:36:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:48Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbe1a4b0f59c namespace: kserve-ci-e2e-test resourceVersion: "18967" uid: dac5f4da-830a-4779-895b-b1d42e0af7c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbf15e31cedd namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: 11ab5fb9-818e-4d23-864a-66d5ab69c0ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: 836f3801-5cf4-468d-9998-99105397f7ab kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-412bf-predictor-556fc6f8-2s6tc.18b2cbf15e32398e namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 71f2f191-f7a2-4a1d-a469-3985f3f68973 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-412bf-predictor-556fc6f8 namespace: kserve-ci-e2e-test resourceVersion: "18482" uid: b4b9ba2a-8667-4a5a-970a-74ba9fa9244e kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'Created pod: success-200-isvc-412bf-predictor-556fc6f8-2s6tc' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-412bf-predictor-556fc6f8.18b2cbe104c50b6e namespace: kserve-ci-e2e-test resourceVersion: "18503" uid: 6f5538eb-0dff-4e58-8207-94a92a87976d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18477" uid: fc6f1e61-ef7b-4be9-9478-e412ce5bbf5b kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: Scaled up replica set success-200-isvc-412bf-predictor-556fc6f8 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-412bf-predictor.18b2cbe101c1d872 namespace: kserve-ci-e2e-test resourceVersion: "18490" uid: bdf03fb7-23f9-412a-9aa8-2ef22a76e63a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18483" uid: 6e8ae773-1377-483c-9711-80fd9b184bad kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: success-200-isvc-412bf-predictor.18b2cbe480fd0f0e namespace: kserve-ci-e2e-test resourceVersion: "18851" uid: 347fc65b-5dd2-4f76-b9a3-6065b68ad4a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18483" uid: 6e8ae773-1377-483c-9711-80fd9b184bad kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: success-200-isvc-412bf-predictor.18b2cbe481722d83 namespace: kserve-ci-e2e-test resourceVersion: "18852" uid: 453b616e-249d-4e5c-a580-f353a8e5f173 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18718" uid: 6e8ae773-1377-483c-9711-80fd9b184bad kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:10Z" name: success-200-isvc-412bf-predictor.18b2cbeb7f486556 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 761325a2-c7dc-46c7-82c6-b9a5243744c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-412bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "18718" uid: 6e8ae773-1377-483c-9711-80fd9b184bad kind: Event lastTimestamp: "2026-05-25T11:37:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:10Z" name: success-200-isvc-412bf-predictor.18b2cbeb7fb3c531 namespace: kserve-ci-e2e-test resourceVersion: "19318" uid: efa42d78-e1e9-472f-8def-b3e4f5bfc3f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "18468" uid: b6a0cc99-6080-4013-9b05-7139f25cf2a7 kind: Event lastTimestamp: "2026-05-25T11:36:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:58Z" name: success-200-isvc-412bf.18b2cbe101b5bb62 namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: aec2db55-8fd8-4d4c-8538-5ff989a83fd3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "18468" uid: b6a0cc99-6080-4013-9b05-7139f25cf2a7 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-412bf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-412bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-412bf.18b2cbe102a79f3f namespace: kserve-ci-e2e-test resourceVersion: "18494" uid: bcd92914-2aa8-49a5-a54a-4feeaff0c56e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "18468" uid: b6a0cc99-6080-4013-9b05-7139f25cf2a7 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-412bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-412bf.18b2cbe102a7f191 namespace: kserve-ci-e2e-test resourceVersion: "18497" uid: fa4c1592-6b4f-4082-ba81-53c1896cdcdc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-412bf namespace: kserve-ci-e2e-test resourceVersion: "19203" uid: b6a0cc99-6080-4013-9b05-7139f25cf2a7 kind: Event lastTimestamp: "2026-05-25T11:36:58Z" message: InferenceService [success-200-isvc-412bf] is Ready metadata: creationTimestamp: "2026-05-25T11:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:58Z" name: success-200-isvc-412bf.18b2cbec6262fbf6 namespace: kserve-ci-e2e-test resourceVersion: "19205" uid: 076480b7-ee31-4644-b635-a5328099d86a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:35:40.925965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18142" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-49543-predictor-75fc554798-2v6w6 to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:35:40Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda3b7ac25c namespace: kserve-ci-e2e-test resourceVersion: "18154" uid: eb5d1521-feff-44c7-aac1-5bdb9a2e73ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18177" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda5572c6db namespace: kserve-ci-e2e-test resourceVersion: "18199" uid: e08296d2-cd39-47e1-846f-9a230c0a5a8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda574a0a5e namespace: kserve-ci-e2e-test resourceVersion: "18201" uid: 2fe0810b-788b-49f5-b81a-69dcbfbbcd52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda5a757fbe namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: 8d671aa9-3d2f-40d8-9803-156da5767e86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda5af15e26 namespace: kserve-ci-e2e-test resourceVersion: "18204" uid: bda1fe0f-f4ac-4c2c-9822-b0cbed39bc20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda5afcfad8 namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: dc0219b1-3a11-43b3-9aee-c58a9f33fd26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda5debc2f8 namespace: kserve-ci-e2e-test resourceVersion: "18206" uid: f54d1b09-9b2f-4807-a0a9-7ba98ab58c35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:35:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:41Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda5e6ae596 namespace: kserve-ci-e2e-test resourceVersion: "18208" uid: 8a010a3f-7bed-45a7-b0de-fba145469273 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:36:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:17Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbda6739b714 namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 443e6683-42d9-4121-949a-848c0cda4c00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbea804a61f1 namespace: kserve-ci-e2e-test resourceVersion: "19031" uid: e50f0792-5560-4e07-bd5e-8522f588458b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:36:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:50Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbea804ad62b namespace: kserve-ci-e2e-test resourceVersion: "19033" uid: 7bc7c7ae-1260-4e7a-bb87-1be5e19892d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49543-predictor-75fc554798-2v6w6 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: b2411ec6-a6aa-478a-8042-b43378fb3912 kind: Event lastTimestamp: "2026-05-25T11:36:52Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:52Z" name: success-200-isvc-49543-predictor-75fc554798-2v6w6.18b2cbeaef5c1c78 namespace: kserve-ci-e2e-test resourceVersion: "19128" uid: 8893e58b-03c6-49bc-8569-2bb4f581c401 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-49543-predictor-75fc554798 namespace: kserve-ci-e2e-test resourceVersion: "18116" uid: 06d46a83-4c51-4434-b8d2-6bef48244bd4 kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: 'Created pod: success-200-isvc-49543-predictor-75fc554798-2v6w6' metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:35:40Z" name: success-200-isvc-49543-predictor-75fc554798.18b2cbda39b77f58 namespace: kserve-ci-e2e-test resourceVersion: "18145" uid: a4499c2c-4286-4e3a-94e4-681cf4afcd11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: 102c6c5a-5dfd-4eb5-9afd-4370992d2427 kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: Scaled up replica set success-200-isvc-49543-predictor-75fc554798 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:35:40Z" name: success-200-isvc-49543-predictor.18b2cbda27388880 namespace: kserve-ci-e2e-test resourceVersion: "18121" uid: 54b9cdb6-618f-466c-b6d7-eee1ef2fe376 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:35:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: d06e4a62-558c-48d3-8621-bf43dbd5f27a kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-49543-predictor.18b2cbdda69e915d namespace: kserve-ci-e2e-test resourceVersion: "18566" uid: fe63eb30-05c6-4e8f-9b8d-fae26306caf0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:35:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: d06e4a62-558c-48d3-8621-bf43dbd5f27a kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:35:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-49543-predictor.18b2cbdda70bae0a namespace: kserve-ci-e2e-test resourceVersion: "18567" uid: 7a188162-94c9-4655-918e-e385883aa52f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: d06e4a62-558c-48d3-8621-bf43dbd5f27a kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: success-200-isvc-49543-predictor.18b2cbe4a54eaf43 namespace: kserve-ci-e2e-test resourceVersion: "18883" uid: 9672480d-a84e-43b7-9970-534552dbc2f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:36:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49543-predictor namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: d06e4a62-558c-48d3-8621-bf43dbd5f27a kind: Event lastTimestamp: "2026-05-25T11:36:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:36:40Z" name: success-200-isvc-49543-predictor.18b2cbe4a5b8d9f9 namespace: kserve-ci-e2e-test resourceVersion: "18884" uid: 13a27671-4544-4a15-ae3a-a320dd04a709 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49543 namespace: kserve-ci-e2e-test resourceVersion: "18099" uid: ec641ef4-c1f6-4aba-b273-882541bd7a9c kind: Event lastTimestamp: "2026-05-25T11:36:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:27Z" name: success-200-isvc-49543.18b2cbda273058a5 namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: 685acefe-5154-413d-ac16-f55b2325ec3d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49543 namespace: kserve-ci-e2e-test resourceVersion: "18752" uid: ec641ef4-c1f6-4aba-b273-882541bd7a9c kind: Event lastTimestamp: "2026-05-25T11:36:27Z" message: InferenceService [success-200-isvc-49543] is Ready metadata: creationTimestamp: "2026-05-25T11:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:36:27Z" name: success-200-isvc-49543.18b2cbe5242fa3d4 namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: 2b2f46da-46a0-446d-b7b7-ec0d815fee01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:26:25.325711Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-55329-predictor-867dcfc56d-6sgzv to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb58df1fa961 namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: 72ef1b8f-1f13-4fd3-a6bf-dcf8beed0702 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-55329-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb58e6d47e66 namespace: kserve-ci-e2e-test resourceVersion: "13788" uid: 92fa5fe4-3fa4-4840-80a4-a7ea510e1117 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13781" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb591ce37175 namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: e74d29c9-8891-4801-a8d2-4907b7752c7f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb591e30b3a6 namespace: kserve-ci-e2e-test resourceVersion: "13834" uid: abcef77d-c7eb-4142-bb7a-0616525266d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb5920dc25ab namespace: kserve-ci-e2e-test resourceVersion: "13835" uid: 0cd97eef-bd4f-4d08-a2b3-3f61b40dd50f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb5921bc3cf2 namespace: kserve-ci-e2e-test resourceVersion: "13836" uid: c64aeb1e-a192-4d4d-b8c2-de0c4ac2e3f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb5921c76e63 namespace: kserve-ci-e2e-test resourceVersion: "13837" uid: 3f9b1e5f-b5ab-472b-9643-b470cdc79225 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb59248e2909 namespace: kserve-ci-e2e-test resourceVersion: "13838" uid: 246317ba-d2a2-4bc7-92d3-ea6b0d586641 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:26:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:26Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb5924ff9a4f namespace: kserve-ci-e2e-test resourceVersion: "13839" uid: 2f9b3f39-0a3a-4bd2-a2ff-bfe97a94134a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:27:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:04Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cb5988038b30 namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: 2f2220a8-fe87-401f-b25d-5aeb9b481972 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:40Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cbda2e476885 namespace: kserve-ci-e2e-test resourceVersion: "18132" uid: 64fde65a-82b2-4639-be9e-ffe39910bdf4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 91b69d67-be24-495e-a758-f093947a9b5b kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:40Z" name: success-200-isvc-55329-predictor-867dcfc56d-6sgzv.18b2cbda2e480259 namespace: kserve-ci-e2e-test resourceVersion: "18133" uid: b849fc7f-f7dc-43eb-9c91-6c44e8359687 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-55329-predictor-867dcfc56d namespace: kserve-ci-e2e-test resourceVersion: "13752" uid: 8f0317b3-5325-4673-8261-efc93caa0ad4 kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: 'Created pod: success-200-isvc-55329-predictor-867dcfc56d-6sgzv' metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-55329-predictor-867dcfc56d.18b2cb58deb909f4 namespace: kserve-ci-e2e-test resourceVersion: "13761" uid: 227b7450-7517-4308-a806-9b13f27bb2b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13750" uid: 51dcdcf6-5304-472c-8727-0e98b3aac29a kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Scaled up replica set success-200-isvc-55329-predictor-867dcfc56d from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:25Z" name: success-200-isvc-55329-predictor.18b2cb58dd1c0cdf namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: 7a60bef7-d63c-4f07-8d60-7a053b919ef5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13751" uid: 7a366de4-3b1c-4c4f-80b1-1f7cf628b226 kind: Event lastTimestamp: "2026-05-25T11:26:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:40Z" name: success-200-isvc-55329-predictor.18b2cb5c5ba0e04b namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: 25b0c135-6c2c-426a-acd4-0b61b455504f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13751" uid: 7a366de4-3b1c-4c4f-80b1-1f7cf628b226 kind: Event lastTimestamp: "2026-05-25T11:26:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:40Z" name: success-200-isvc-55329-predictor.18b2cb5c5c12af2f namespace: kserve-ci-e2e-test resourceVersion: "13974" uid: 0b0a3416-b10a-460b-9145-05075ad897b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 7a366de4-3b1c-4c4f-80b1-1f7cf628b226 kind: Event lastTimestamp: "2026-05-25T11:27:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:40Z" name: success-200-isvc-55329-predictor.18b2cb5fdb021ddd namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: b6bc7baf-2e40-4034-be27-d80be0824d02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55329-predictor namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 7a366de4-3b1c-4c4f-80b1-1f7cf628b226 kind: Event lastTimestamp: "2026-05-25T11:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:40Z" name: success-200-isvc-55329-predictor.18b2cb5fdb6e226a namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: ca183140-4a7a-4dd7-a1d2-0e0e96479729 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-55329 namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: 6452ac56-b2b9-4572-a681-a924c496fca5 kind: Event lastTimestamp: "2026-05-25T11:27:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:14Z" name: success-200-isvc-55329.18b2cb58dcdda18c namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 9bb44ebc-837b-4ddc-b32a-a55d7bcb7d99 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-55329 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 6452ac56-b2b9-4572-a681-a924c496fca5 kind: Event lastTimestamp: "2026-05-25T11:27:14Z" message: InferenceService [success-200-isvc-55329] is Ready metadata: creationTimestamp: "2026-05-25T11:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:14Z" name: success-200-isvc-55329.18b2cb644331460f namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 9b22d448-6ae9-4b12-919f-df105217ff4a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:37:20.326394Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b59e4-predictor-7dfbdf574-768rn to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf16034d7ec namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 2f5a6f9d-e6c8-4cf6-8488-78d3f373b8a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b59e4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf16bdb58de namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: f275e421-9860-4a0e-9bdb-9a331c9d51a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf19d36aa22 namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 81b89838-2974-4b5f-818b-8b7e6e2976c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf19e91cc23 namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: d6a38f5c-ab9f-462f-93b2-adaae913b7aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf1a1ada9f6 namespace: kserve-ci-e2e-test resourceVersion: "19553" uid: 3cbee47d-99c9-4868-b38f-8659b407a455 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf1a21eab6f namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 70d16a89-3667-4985-acd5-ddea5f7e605d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf1a22b2df3 namespace: kserve-ci-e2e-test resourceVersion: "19555" uid: f0ce6b03-38cf-4471-9b96-388d153b8c63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf1a50748b2 namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 1c0b82f0-b93b-495b-9496-c766ab220fbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:21Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf1a57aa1c3 namespace: kserve-ci-e2e-test resourceVersion: "19559" uid: a31c6a74-ab43-4cf4-895a-170493a2cce1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:37:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:37:58Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cbf1fd8e77e4 namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: d5d2fcd8-357a-4efd-becb-1d79413c7152 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cc728b8db8a1 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: f49ae5a7-17dc-4930-a36b-c0e3fb8e22a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 1c459914-3782-4a1d-a188-2a60021bb906 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-b59e4-predictor-7dfbdf574-768rn.18b2cc728b8e69ab namespace: kserve-ci-e2e-test resourceVersion: "23748" uid: 2cd0e852-8839-401a-bf37-1e6879b2359b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b59e4-predictor-7dfbdf574 namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: 790dea61-7445-437b-860c-b7c61a7701b2 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'Created pod: success-200-isvc-b59e4-predictor-7dfbdf574-768rn' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-b59e4-predictor-7dfbdf574.18b2cbf15f3b5eab namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: 90693733-e88d-495a-9d03-3bd8d4055c7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19452" uid: 4b041b36-2d0f-4069-aa49-c194ef4444d5 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: Scaled up replica set success-200-isvc-b59e4-predictor-7dfbdf574 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-b59e4-predictor.18b2cbf15c446519 namespace: kserve-ci-e2e-test resourceVersion: "19464" uid: eee6da25-5fe6-4764-8c4d-0f3ea017892a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 68951af8-0c65-43a2-b0e8-98a185f9b366 kind: Event lastTimestamp: "2026-05-25T11:37:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:50Z" name: success-200-isvc-b59e4-predictor.18b2cbf4daf99c30 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 515f244c-9e85-4cf9-8c9c-bdac703b0edd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 68951af8-0c65-43a2-b0e8-98a185f9b366 kind: Event lastTimestamp: "2026-05-25T11:37:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:37:50Z" name: success-200-isvc-b59e4-predictor.18b2cbf4db6b688f namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 3c1fedb4-2064-484a-a61d-339ac7026858 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 68951af8-0c65-43a2-b0e8-98a185f9b366 kind: Event lastTimestamp: "2026-05-25T11:38:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:35Z" name: success-200-isvc-b59e4-predictor.18b2cbfbd994411d namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8a9c5d34-d2bc-4beb-9ec5-60040bb9e499 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b59e4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 68951af8-0c65-43a2-b0e8-98a185f9b366 kind: Event lastTimestamp: "2026-05-25T11:38:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:38:35Z" name: success-200-isvc-b59e4-predictor.18b2cbfbda045129 namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: ba7eb812-3455-41ad-96c6-1b98f3d8b73c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 378e641d-1853-48c5-863f-d210186baa83 kind: Event lastTimestamp: "2026-05-25T11:38:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:38:09Z" name: success-200-isvc-b59e4.18b2cbf15c17e8e9 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: b30499e5-54f3-4c49-8289-b078d6a5e487 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 378e641d-1853-48c5-863f-d210186baa83 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'Failed to update status for InferenceService "success-200-isvc-b59e4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b59e4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-b59e4.18b2cbf15cfec8cd namespace: kserve-ci-e2e-test resourceVersion: "19469" uid: 57b5dd16-46a1-4fb2-9acf-cfeb94ffe7e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 378e641d-1853-48c5-863f-d210186baa83 kind: Event lastTimestamp: "2026-05-25T11:37:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b59e4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:37:20Z" name: success-200-isvc-b59e4.18b2cbf15cff38df namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: b28138a6-4f2f-41d1-ab63-5f47598ce1e4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:38:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b59e4 namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 378e641d-1853-48c5-863f-d210186baa83 kind: Event lastTimestamp: "2026-05-25T11:38:09Z" message: InferenceService [success-200-isvc-b59e4] is Ready metadata: creationTimestamp: "2026-05-25T11:38:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:38:09Z" name: success-200-isvc-b59e4.18b2cbfcb89bdb80 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: dd155dbd-a382-44c2-bc69-596886a1eccf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:46:35.133632Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23747" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc728d4b75fa namespace: kserve-ci-e2e-test resourceVersion: "23754" uid: aae53304-3708-434c-8fc3-8c0b593783b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23757" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72a6badb00 namespace: kserve-ci-e2e-test resourceVersion: "23807" uid: 540038c4-e0ca-4bcc-82bd-b7d1465e5760 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72a8325950 namespace: kserve-ci-e2e-test resourceVersion: "23809" uid: 3a0013f2-47b2-469b-8d28-f338de3c6b0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72ab11e025 namespace: kserve-ci-e2e-test resourceVersion: "23810" uid: fb8f9db1-301e-40ec-84be-fda874c37929 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72aba5fcd1 namespace: kserve-ci-e2e-test resourceVersion: "23811" uid: 21d55521-941b-4318-8cec-8ff817bbccbd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72abb2f4bd namespace: kserve-ci-e2e-test resourceVersion: "23812" uid: 3607457f-c0ce-456e-a175-50563a03f068 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72b03f1a7a namespace: kserve-ci-e2e-test resourceVersion: "23817" uid: 96ca80f1-c5bb-421b-b076-399e7c86f897 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc72b0b961d7 namespace: kserve-ci-e2e-test resourceVersion: "23818" uid: 2331e409-b43e-42b2-9e74-9fe27e30c7b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T11:47:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:13Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cc731e9dca26 namespace: kserve-ci-e2e-test resourceVersion: "24211" uid: 6df6b6c1-bd3d-4b31-b3c5-5c96ed3d6148 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T12:03:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:03:54Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cd648adf8212 namespace: kserve-ci-e2e-test resourceVersion: "31777" uid: 39184bcf-f4bb-43cb-b667-e714c359fbcd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 38d5c0a0-73a1-4d64-84ec-54ef72df2661 kind: Event lastTimestamp: "2026-05-25T12:03:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:03:54Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r.18b2cd648ae03eaa namespace: kserve-ci-e2e-test resourceVersion: "31778" uid: ae717f28-0cf5-4ba4-9b78-dfe8aa55c662 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c1cab-predictor-7774d7cc9b namespace: kserve-ci-e2e-test resourceVersion: "23733" uid: 6d2f7555-2366-4ae9-863a-d94935d697f6 kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'Created pod: success-200-isvc-c1cab-predictor-7774d7cc9b-qvk9r' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor-7774d7cc9b.18b2cc728cb93640 namespace: kserve-ci-e2e-test resourceVersion: "23750" uid: d53f64f2-f042-4791-82cc-0ca365640fb0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23729" uid: 0389e809-e351-4259-8b89-bfe930ccb59a kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: Scaled up replica set success-200-isvc-c1cab-predictor-7774d7cc9b from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab-predictor.18b2cc728a48a1b5 namespace: kserve-ci-e2e-test resourceVersion: "23737" uid: 5a7e5809-6b32-43f7-a938-f6c020baa174 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23734" uid: 98c32e50-8c5a-4a13-9553-2fbced68c79c kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: success-200-isvc-c1cab-predictor.18b2cc760a0d57c0 namespace: kserve-ci-e2e-test resourceVersion: "24099" uid: c49802d7-7b1d-45f8-9c4e-7463727123ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23734" uid: 98c32e50-8c5a-4a13-9553-2fbced68c79c kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: success-200-isvc-c1cab-predictor.18b2cc760ad73b0c namespace: kserve-ci-e2e-test resourceVersion: "24100" uid: 9a126e3b-543d-4d3a-bacc-372cbcbc9e50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23962" uid: 98c32e50-8c5a-4a13-9553-2fbced68c79c kind: Event lastTimestamp: "2026-05-25T11:47:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:50Z" name: success-200-isvc-c1cab-predictor.18b2cc7d08814fab namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: d52d5d78-c02c-4d24-89d8-e3bd5b324d8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1cab-predictor namespace: kserve-ci-e2e-test resourceVersion: "23962" uid: 98c32e50-8c5a-4a13-9553-2fbced68c79c kind: Event lastTimestamp: "2026-05-25T11:47:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:50Z" name: success-200-isvc-c1cab-predictor.18b2cc7d09125b2f namespace: kserve-ci-e2e-test resourceVersion: "24709" uid: 2bd99032-c48d-4957-8e60-25323b55425c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "23714" uid: e917e82c-02cb-4e06-95bd-1ebc722510bc kind: Event lastTimestamp: "2026-05-25T11:47:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:23Z" name: success-200-isvc-c1cab.18b2cc728a5528de namespace: kserve-ci-e2e-test resourceVersion: "24440" uid: 1915d89c-42b1-4b21-83e6-8c3ba91309bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "23714" uid: e917e82c-02cb-4e06-95bd-1ebc722510bc kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-c1cab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c1cab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab.18b2cc728b138a0e namespace: kserve-ci-e2e-test resourceVersion: "23743" uid: 1d51e0fd-20e6-4a21-8e1b-09e0d85d1698 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "23714" uid: e917e82c-02cb-4e06-95bd-1ebc722510bc kind: Event lastTimestamp: "2026-05-25T11:46:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c1cab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:35Z" name: success-200-isvc-c1cab.18b2cc728b13dd5a namespace: kserve-ci-e2e-test resourceVersion: "23744" uid: 8e8853c1-5ed6-4d60-a2ed-31b035e13d66 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24438" uid: e917e82c-02cb-4e06-95bd-1ebc722510bc kind: Event lastTimestamp: "2026-05-25T11:47:23Z" message: InferenceService [success-200-isvc-c1cab] is Ready metadata: creationTimestamp: "2026-05-25T11:47:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:23Z" name: success-200-isvc-c1cab.18b2cc7dd9b1ea7f namespace: kserve-ci-e2e-test resourceVersion: "24439" uid: ebb8ad86-2fe1-4eec-b8a2-e334e274367e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:46:05.846287Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23399" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bbba21461 namespace: kserve-ci-e2e-test resourceVersion: "23408" uid: 9e0fd5c0-8c19-463f-8f38-574db3fddd84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c3e68-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bc3f20618 namespace: kserve-ci-e2e-test resourceVersion: "23436" uid: 8a1f6144-da3a-4ee2-9129-33875503e726 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23417" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:46:06Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bf8c624b5 namespace: kserve-ci-e2e-test resourceVersion: "23451" uid: a0559bd4-15c7-4333-9b20-7b12dc668586 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:06Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bfa4a35e2 namespace: kserve-ci-e2e-test resourceVersion: "23453" uid: ce1fd6ee-e353-400e-b5fb-e2f80751dee8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:06Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bfdf1caba namespace: kserve-ci-e2e-test resourceVersion: "23458" uid: e07e4784-1d9b-4b2e-ab15-549044c28ca3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:06Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bfe678e0b namespace: kserve-ci-e2e-test resourceVersion: "23459" uid: 3434bb8f-8a9b-431d-8477-dcebc3392b2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:06Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6bfe73d6a2 namespace: kserve-ci-e2e-test resourceVersion: "23460" uid: 6ca28baf-c099-40a9-8ac0-d28a3295e553 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6c01627306 namespace: kserve-ci-e2e-test resourceVersion: "23463" uid: cb03601e-cc36-49f2-8dd1-9ffb16983c75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:07Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6c01d480f7 namespace: kserve-ci-e2e-test resourceVersion: "23465" uid: 6066b4ab-78c4-4f4f-8bbf-c85d3481e032 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:46:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:46:44Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc6c59067fc5 namespace: kserve-ci-e2e-test resourceVersion: "23919" uid: cf323f05-45cb-4de8-9c59-b57b451a69cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc7c212834d4 namespace: kserve-ci-e2e-test resourceVersion: "24282" uid: e28f6767-5ef7-41b2-8d09-bce3b2057aee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt namespace: kserve-ci-e2e-test resourceVersion: "23403" uid: ef0629fb-809f-47ab-8312-7f97a370c16b kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt.18b2cc7c2128a36e namespace: kserve-ci-e2e-test resourceVersion: "24287" uid: e84f3d2e-025d-47a9-84c0-dbd71a59c902 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3e68-predictor-6f8fff67db namespace: kserve-ci-e2e-test resourceVersion: "23375" uid: 79741ebc-084a-451c-a150-35d381d17e1c kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'Created pod: success-200-isvc-c3e68-predictor-6f8fff67db-sf4vt' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-c3e68-predictor-6f8fff67db.18b2cc6bbb00a46f namespace: kserve-ci-e2e-test resourceVersion: "23401" uid: 58efd8de-f19b-4a87-b90d-8d4a55e6db83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23372" uid: 3e274e29-8b8d-4cad-8b06-dbe3e2799044 kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: Scaled up replica set success-200-isvc-c3e68-predictor-6f8fff67db from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-c3e68-predictor.18b2cc6bb69a7cbd namespace: kserve-ci-e2e-test resourceVersion: "23378" uid: 81668f93-42cb-4aa2-a3bf-51beaeff596b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23374" uid: 33d38dd5-a57e-495a-8150-374ebe7fa1bf kind: Event lastTimestamp: "2026-05-25T11:46:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:20Z" name: success-200-isvc-c3e68-predictor.18b2cc6f35dfa61e namespace: kserve-ci-e2e-test resourceVersion: "23587" uid: f20a0e93-408c-4f3f-9dd5-a249b0dbb00d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23374" uid: 33d38dd5-a57e-495a-8150-374ebe7fa1bf kind: Event lastTimestamp: "2026-05-25T11:46:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:46:20Z" name: success-200-isvc-c3e68-predictor.18b2cc6f36672d6e namespace: kserve-ci-e2e-test resourceVersion: "23588" uid: dee898ec-2c92-41fa-9d82-8231daa13acc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23586" uid: 33d38dd5-a57e-495a-8150-374ebe7fa1bf kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: success-200-isvc-c3e68-predictor.18b2cc72b521510e namespace: kserve-ci-e2e-test resourceVersion: "24118" uid: c79a214c-3c22-475d-bd69-d8db066d238b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T11:46:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3e68-predictor namespace: kserve-ci-e2e-test resourceVersion: "23586" uid: 33d38dd5-a57e-495a-8150-374ebe7fa1bf kind: Event lastTimestamp: "2026-05-25T11:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:05Z" name: success-200-isvc-c3e68-predictor.18b2cc72b596dfe8 namespace: kserve-ci-e2e-test resourceVersion: "24121" uid: 7aa7916e-49a4-4576-8d9e-2ec95e6df670 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "23350" uid: 57296e13-edbf-40c9-9851-41d88599c524 kind: Event lastTimestamp: "2026-05-25T11:46:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:54Z" name: success-200-isvc-c3e68.18b2cc6bb66cb127 namespace: kserve-ci-e2e-test resourceVersion: "24014" uid: 5e57b4fb-927e-4606-ba6b-da6fafeadd1c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "23350" uid: 57296e13-edbf-40c9-9851-41d88599c524 kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'Failed to update status for InferenceService "success-200-isvc-c3e68": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3e68": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-c3e68.18b2cc6bb6c02cfe namespace: kserve-ci-e2e-test resourceVersion: "23380" uid: c5ec9a67-2d98-46fc-9e86-c0914eead86f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "23350" uid: 57296e13-edbf-40c9-9851-41d88599c524 kind: Event lastTimestamp: "2026-05-25T11:46:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3e68": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:05Z" name: success-200-isvc-c3e68.18b2cc6bb6c0732a namespace: kserve-ci-e2e-test resourceVersion: "23381" uid: 6ebe6ad1-427e-444d-ac44-f4041288bdad reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:46:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3e68 namespace: kserve-ci-e2e-test resourceVersion: "24012" uid: 57296e13-edbf-40c9-9851-41d88599c524 kind: Event lastTimestamp: "2026-05-25T11:46:54Z" message: InferenceService [success-200-isvc-c3e68] is Ready metadata: creationTimestamp: "2026-05-25T11:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:46:54Z" name: success-200-isvc-c3e68.18b2cc77143dd564 namespace: kserve-ci-e2e-test resourceVersion: "24013" uid: 057b75b0-6d76-4d62-81fb-0d214740e87e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:47:16.302420Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24280" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c23255958 namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 2d0e72a4-88bb-48a6-a285-47dfd675774d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24291" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c3c616e42 namespace: kserve-ci-e2e-test resourceVersion: "24343" uid: 795c3301-587b-40da-bc81-6f6e5512cdec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c3e4c15d1 namespace: kserve-ci-e2e-test resourceVersion: "24347" uid: 01431c06-f26e-418b-9fb8-d5702d1bc436 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c40f8813d namespace: kserve-ci-e2e-test resourceVersion: "24348" uid: 8b3da6ea-044e-416b-9013-2b374a767e10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c416db256 namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 0931d909-94fb-4b72-8bdb-2f4c8adbd5b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c4178c573 namespace: kserve-ci-e2e-test resourceVersion: "24350" uid: 8f875664-1c86-44c3-8797-83d78e8e2814 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c4405f2f3 namespace: kserve-ci-e2e-test resourceVersion: "24351" uid: 49f93144-323a-4eca-b714-588e90a38d69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7c4478a9ec namespace: kserve-ci-e2e-test resourceVersion: "24352" uid: 45b8e25d-33a4-41ca-ba65-ab62298239af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:47:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:54Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2cc7cb13e323d namespace: kserve-ci-e2e-test resourceVersion: "24742" uid: 0055d5a1-c0c4-4182-a85d-2f02caaca522 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:56:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:56:31Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2ccfd49e7f737 namespace: kserve-ci-e2e-test resourceVersion: "28695" uid: 04e80c0b-e67d-472f-a9b0-0771af9e1d5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:56:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: d68071cf-45b3-407d-a0b0-489687f61b85 kind: Event lastTimestamp: "2026-05-25T11:56:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:56:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:56:31Z" name: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2.18b2ccfd49e88a1a namespace: kserve-ci-e2e-test resourceVersion: "28697" uid: 74f5d354-e68d-4764-bee4-1a7d83e131bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c7f41-predictor-696877f5fd namespace: kserve-ci-e2e-test resourceVersion: "24266" uid: 69c3a57d-b758-4f22-8cdf-e1374dfb3427 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'Created pod: success-200-isvc-c7f41-predictor-696877f5fd-l8mq2' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor-696877f5fd.18b2cc7c21dc7298 namespace: kserve-ci-e2e-test resourceVersion: "24285" uid: f557fa9e-4356-4d1d-bd8d-cc4a455cb03b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24261" uid: 0da8156a-2df0-4ebc-a977-41c9dad9f915 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: Scaled up replica set success-200-isvc-c7f41-predictor-696877f5fd from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41-predictor.18b2cc7c1de36346 namespace: kserve-ci-e2e-test resourceVersion: "24272" uid: 8f5a016e-0e30-4ce4-9187-0d0c42b65934 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24265" uid: fd74f686-5306-4f76-b806-85fc570f3e67 kind: Event lastTimestamp: "2026-05-25T11:47:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:46Z" name: success-200-isvc-c7f41-predictor.18b2cc7f9c67ec00 namespace: kserve-ci-e2e-test resourceVersion: "24672" uid: fa5258b2-cab9-4bb8-b1c2-387517880fcf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:47:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24265" uid: fd74f686-5306-4f76-b806-85fc570f3e67 kind: Event lastTimestamp: "2026-05-25T11:47:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:46Z" name: success-200-isvc-c7f41-predictor.18b2cc7f9cd1c6f6 namespace: kserve-ci-e2e-test resourceVersion: "24673" uid: 75137f05-20ec-4b85-9122-c4379af33d24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24509" uid: fd74f686-5306-4f76-b806-85fc570f3e67 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:16Z" name: success-200-isvc-c7f41-predictor.18b2cc869b2c6fec namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: eeadcc6d-59c5-4255-a696-82decdc31c58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:48:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7f41-predictor namespace: kserve-ci-e2e-test resourceVersion: "24509" uid: fd74f686-5306-4f76-b806-85fc570f3e67 kind: Event lastTimestamp: "2026-05-25T11:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:16Z" name: success-200-isvc-c7f41-predictor.18b2cc869b9d7f36 namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: 2fe8bf5d-a2e3-4981-a46c-d8b950f22c40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24249" uid: c8b7cc3c-d6fa-4d39-a294-66e3bdb96ed3 kind: Event lastTimestamp: "2026-05-25T11:48:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:48:04Z" name: success-200-isvc-c7f41.18b2cc7c1d91a953 namespace: kserve-ci-e2e-test resourceVersion: "24826" uid: 739fc32e-24d7-46f5-8c4b-8bf2b1c94d0c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24249" uid: c8b7cc3c-d6fa-4d39-a294-66e3bdb96ed3 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-c7f41": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c7f41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41.18b2cc7c1e6ebf0d namespace: kserve-ci-e2e-test resourceVersion: "24274" uid: e64c55cd-bc3c-4dc3-80c5-e521bf177188 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24249" uid: c8b7cc3c-d6fa-4d39-a294-66e3bdb96ed3 kind: Event lastTimestamp: "2026-05-25T11:47:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c7f41": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:16Z" name: success-200-isvc-c7f41.18b2cc7c1e6f065a namespace: kserve-ci-e2e-test resourceVersion: "24279" uid: dff40f74-ac02-462a-b34c-a7d659ba7825 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:48:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7f41 namespace: kserve-ci-e2e-test resourceVersion: "24823" uid: c8b7cc3c-d6fa-4d39-a294-66e3bdb96ed3 kind: Event lastTimestamp: "2026-05-25T11:48:04Z" message: InferenceService [success-200-isvc-c7f41] is Ready metadata: creationTimestamp: "2026-05-25T11:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:48:04Z" name: success-200-isvc-c7f41.18b2cc876bff2909 namespace: kserve-ci-e2e-test resourceVersion: "24824" uid: b775a549-12c1-4160-829e-cc93841d1b74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:26:55.188823Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:26:55Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb5fd31a8e5a namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 254a691b-03f1-4232-b1eb-c6ed5122d5f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f16f5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:55Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb5fddabeddc namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 0693db6a-0ac9-48c2-a998-c9f36c42df67 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb601182623b namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 2c175bb5-767d-4484-bf3e-b719bdcd5f0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1534" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb6012ed58d5 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 2de8abb0-c24d-4f1e-a9bd-2deecaa2bf75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb601648a274 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 4e44175d-d8b3-4d8f-be28-970383574556 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb6016c1209e namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 930501c9-7999-459b-b785-4ef652bb6c4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb6016ccb1ad namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: a6cfc0b6-acad-45e4-a037-7c0514ed9614 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb601a21a44a namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 24df0cd9-a740-4b14-a16e-abe8b7c423c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:26:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:56Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb601a9919b9 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 2898b3a1-3854-480a-9eea-557b441f3746 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:27:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T11:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:34Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cb608a07f240 namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 7deceec7-f20a-44c1-a9e9-9d670920c910 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cbe10380927b namespace: kserve-ci-e2e-test resourceVersion: "18498" uid: d49f0d1c-00a2-4faa-a1f5-e7580d2893c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:36:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: d694fff0-c01e-41f2-8e8f-0fb8fb79cb97 kind: Event lastTimestamp: "2026-05-25T11:36:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T11:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:10Z" name: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs.18b2cbe10381f160 namespace: kserve-ci-e2e-test resourceVersion: "18501" uid: 41a18b89-dcde-459a-b045-90dbb1833190 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f16f5-predictor-db96fdd8d namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: b20da2bb-359d-4162-8617-c205b17a2b01 kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: 'Created pod: success-200-isvc-f16f5-predictor-db96fdd8d-nbpfs' metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: success-200-isvc-f16f5-predictor-db96fdd8d.18b2cb5fd2ba313f namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 54c7f2b0-2c15-40c8-a11b-923ed2a86507 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 087dbe9f-8be8-4b05-8bb1-811f028f602e kind: Event lastTimestamp: "2026-05-25T11:26:55Z" message: Scaled up replica set success-200-isvc-f16f5-predictor-db96fdd8d from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:55Z" name: success-200-isvc-f16f5-predictor.18b2cb5fd017134e namespace: kserve-ci-e2e-test resourceVersion: "14201" uid: a3cc394d-932e-4a67-8a32-50ada4c9a5f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: e3e2727b-da0e-4a27-becd-63402681d2ad kind: Event lastTimestamp: "2026-05-25T11:27:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:10Z" name: success-200-isvc-f16f5-predictor.18b2cb634ead9eae namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: e1cf3ae4-7bb2-470a-b1eb-61623e6988d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: e3e2727b-da0e-4a27-becd-63402681d2ad kind: Event lastTimestamp: "2026-05-25T11:27:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:10Z" name: success-200-isvc-f16f5-predictor.18b2cb634f20dadc namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 7c9578cc-0636-41c3-8337-c39fc6cfcb14 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: e3e2727b-da0e-4a27-becd-63402681d2ad kind: Event lastTimestamp: "2026-05-25T11:28:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:25Z" name: success-200-isvc-f16f5-predictor.18b2cb66ce6da07d namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 912f1d1a-bc47-414f-a5d5-5724c228382a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:27:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f16f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: e3e2727b-da0e-4a27-becd-63402681d2ad kind: Event lastTimestamp: "2026-05-25T11:28:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:28:25Z" name: success-200-isvc-f16f5-predictor.18b2cb66cedd6fe3 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 5b351929-9a55-4f74-afdb-c9d408fe13a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T11:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 6dabe024-0ed9-4ef0-82cf-69ca6cdfbb6b kind: Event lastTimestamp: "2026-05-25T11:27:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T11:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:44Z" name: success-200-isvc-f16f5.18b2cb5fcfde94c4 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: de747674-385e-4a20-b898-018cfcfab931 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f16f5 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 6dabe024-0ed9-4ef0-82cf-69ca6cdfbb6b kind: Event lastTimestamp: "2026-05-25T11:27:44Z" message: InferenceService [success-200-isvc-f16f5] is Ready metadata: creationTimestamp: "2026-05-25T11:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:44Z" name: success-200-isvc-f16f5.18b2cb6b46549ec3 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 4d58011e-f8f9-4872-9491-eebff0871e2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:26:14.960817Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-21751-5984bd7768-fbhlq to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:26:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:26:14Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb567553ee21 namespace: kserve-ci-e2e-test resourceVersion: "13582" uid: 84f80b2b-689e-4362-85c7-1d852f86a039 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13586" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:15Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:26:15Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb569037fc4e namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 11b9133c-d5d0-4552-9ac8-3bcdec13f0fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-21751} kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:15Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" metadata: creationTimestamp: "2026-05-25T11:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:15Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb56917fff57 namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: b603edb6-1b53-466b-81ba-27850eaec7db reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-21751} kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" in 2.297s (2.297s including waiting). Image size: 218005276 bytes.' metadata: creationTimestamp: "2026-05-25T11:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:17Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb571a6a8d7b namespace: kserve-ci-e2e-test resourceVersion: "13615" uid: aab3d012-fd83-41e6-b4ee-9096035a6800 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-21751} kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:17Z" message: 'Created container: switch-graph-21751' metadata: creationTimestamp: "2026-05-25T11:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:17Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb571dc18de6 namespace: kserve-ci-e2e-test resourceVersion: "13616" uid: 627e9596-5fde-42aa-810d-cde81fa8e9ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-21751} kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:17Z" message: Started container switch-graph-21751 metadata: creationTimestamp: "2026-05-25T11:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:17Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb571e45612a namespace: kserve-ci-e2e-test resourceVersion: "13617" uid: ac333099-66fc-489a-8e08-e934f20d6fa8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-21751} kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:25Z" message: Stopping container switch-graph-21751 metadata: creationTimestamp: "2026-05-25T11:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:25Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb58d40dceb6 namespace: kserve-ci-e2e-test resourceVersion: "13718" uid: c1c02e8c-ee14-41cd-8bc7-3716c6f51509 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:26:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-21751} kind: Pod name: switch-graph-21751-5984bd7768-fbhlq namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d9e728b7-eaf4-4c88-8b71-c679395cec9c kind: Event lastTimestamp: "2026-05-25T11:26:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:26:54Z" name: switch-graph-21751-5984bd7768-fbhlq.18b2cb59c1ecf1d3 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: a6d8a443-918e-4358-aeb0-adc16d9cd04a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-21751-5984bd7768 namespace: kserve-ci-e2e-test resourceVersion: "13574" uid: e10f4323-0b54-4d58-8afb-8ebb294f8a3c kind: Event lastTimestamp: "2026-05-25T11:26:14Z" message: 'Created pod: switch-graph-21751-5984bd7768-fbhlq' metadata: creationTimestamp: "2026-05-25T11:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:14Z" name: switch-graph-21751-5984bd7768.18b2cb5674d90b4c namespace: kserve-ci-e2e-test resourceVersion: "13578" uid: ce6c197c-1b0c-4d1d-8b6c-39c5fa597843 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-21751 namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 276791b4-eef0-4ed1-9cd2-9352b5841fbf kind: Event lastTimestamp: "2026-05-25T11:26:14Z" message: Scaled up replica set switch-graph-21751-5984bd7768 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:26:14Z" name: switch-graph-21751.18b2cb5674161627 namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: b5bc1ec9-cdf2-436e-830f-f1c1dc162230 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:26:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-21751 namespace: kserve-ci-e2e-test resourceVersion: "13590" uid: 74bed689-08a5-435e-9c1c-d59ec03d7cc5 kind: Event lastTimestamp: "2026-05-25T11:26:15Z" message: InferenceGraph [switch-graph-21751] is Ready metadata: creationTimestamp: "2026-05-25T11:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:26:15Z" name: switch-graph-21751.18b2cb567973f897 namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: f00b5c27-2723-459f-a7f3-afb90b39a441 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:27:25.370789Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-55329-668cc95c65-m2767 to ip-10-0-141-33.ec2.internal metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:27:25Z" name: switch-graph-55329-668cc95c65-m2767.18b2cb66da16c001 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 4c05e23a-156e-481d-b84b-dfac08b8ac97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:27:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-55329-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:25Z" name: switch-graph-55329-668cc95c65-m2767.18b2cb66e12afad0 namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: d8588bdc-7f52-4980-805e-31833452aa25 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:26Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:27:26Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:27:26Z" name: switch-graph-55329-668cc95c65-m2767.18b2cb67173ad927 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 7d5ea68d-777c-46da-b3cf-f1b0ebd0bee7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-55329} kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:27:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:26Z" name: switch-graph-55329-668cc95c65-m2767.18b2cb67186f3ff9 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: b039baef-36eb-4c5c-8fa0-735798b487f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-55329} kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:27:26Z" message: 'Created container: switch-graph-55329' metadata: creationTimestamp: "2026-05-25T11:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:26Z" name: switch-graph-55329-668cc95c65-m2767.18b2cb671b612c29 namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 8a696128-a71e-4fdc-8628-648102b5588f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-55329} kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:27:26Z" message: Started container switch-graph-55329 metadata: creationTimestamp: "2026-05-25T11:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:27:26Z" name: switch-graph-55329-668cc95c65-m2767.18b2cb671bd444ba namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: 5e8f6a75-f515-4876-9300-56be4a2cb955 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:35:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-55329} kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:35:40Z" message: Stopping container switch-graph-55329 metadata: creationTimestamp: "2026-05-25T11:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:35:40Z" name: switch-graph-55329-668cc95c65-m2767.18b2cbda16ead98e namespace: kserve-ci-e2e-test resourceVersion: "18096" uid: c29a1d74-f953-4216-8856-d41724a95961 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T11:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-55329} kind: Pod name: switch-graph-55329-668cc95c65-m2767 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: c93fcb61-4aaf-4d32-a09a-1f8a2cc18457 kind: Event lastTimestamp: "2026-05-25T11:36:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T11:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:36:08Z" name: switch-graph-55329-668cc95c65-m2767.18b2cbdacc1d1f51 namespace: kserve-ci-e2e-test resourceVersion: "18436" uid: b479ab0b-8702-4ad2-97ab-a2474dffa282 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-33.ec2.internal source: component: kubelet host: ip-10-0-141-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-55329-668cc95c65 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: d50c72d6-d605-41e4-9764-c208f0ab7e24 kind: Event lastTimestamp: "2026-05-25T11:27:25Z" message: 'Created pod: switch-graph-55329-668cc95c65-m2767' metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:25Z" name: switch-graph-55329-668cc95c65.18b2cb66d98cf049 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 4b6a56d7-da3c-42d6-a735-e4e8fbe96a6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-55329 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 489100f5-0e96-4ce4-885d-b553810d99ea kind: Event lastTimestamp: "2026-05-25T11:27:25Z" message: Scaled up replica set switch-graph-55329-668cc95c65 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:25Z" name: switch-graph-55329.18b2cb66d8d4c49d namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: 079fc6aa-aacf-4ef7-a284-9cc36362d693 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:27:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-55329 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: addcfa88-c063-4c1f-8ca2-d14745f1ab54 kind: Event lastTimestamp: "2026-05-25T11:27:25Z" message: InferenceGraph [switch-graph-55329] is Ready metadata: creationTimestamp: "2026-05-25T11:27:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:27:25Z" name: switch-graph-55329.18b2cb66de194288 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: b7399dba-82d3-4dd0-a9c1-c716b9a2235d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-55329 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 133e03bf-bfb1-4819-ab14-1d284bd7c52f kind: Event lastTimestamp: "2026-05-25T11:27:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:55Z" name: switch-graph-55329.18b2cb6a5786e78b namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 5d5b63bb-d982-4c1c-bbd4-a158b5043bef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-55329 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 133e03bf-bfb1-4819-ab14-1d284bd7c52f kind: Event lastTimestamp: "2026-05-25T11:27:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:27:55Z" name: switch-graph-55329.18b2cb6a57ea3970 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: cc8a4702-bb4c-489f-8a8d-7bb0acb0cda7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T11:47:35.158828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24557" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c1cab-68466fbf77-hzqld to ip-10-0-131-36.ec2.internal metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cc8087135282 namespace: kserve-ci-e2e-test resourceVersion: "24562" uid: 7e6fa607-11ea-469b-9e92-6d813da5a39e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c55cb6554-tzz8v source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24567" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cc80a03c2e4d namespace: kserve-ci-e2e-test resourceVersion: "24582" uid: 4447c2c6-baeb-467e-893e-087f905bea18 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c1cab} kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9d72397cc063a25c55c85283761fa175ad248411370e06d82ff2601a149a916d" already present on machine metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cc80a1859fea namespace: kserve-ci-e2e-test resourceVersion: "24584" uid: 934ac5b7-0dda-4b66-b5a4-5d5d55f9b860 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c1cab} kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: 'Created container: switch-graph-c1cab' metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cc80a48b7519 namespace: kserve-ci-e2e-test resourceVersion: "24585" uid: 1edeaa42-3bab-42a5-9ecd-82ffa526f420 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c1cab} kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: Started container switch-graph-c1cab metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cc80a4f7297d namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: 3c914614-8725-441b-8932-72716cb0ef85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c1cab} kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: "2026-05-25T12:03:54Z" message: Stopping container switch-graph-c1cab metadata: creationTimestamp: "2026-05-25T12:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:03:54Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cd6482ac1454 namespace: kserve-ci-e2e-test resourceVersion: "31762" uid: 0d0ffbf8-64b6-495a-bf1c-6ebc899f4b6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:03:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c1cab} kind: Pod name: switch-graph-c1cab-68466fbf77-hzqld namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 9a0afa2d-7e9d-4c80-a42a-f69a8370c34e kind: Event lastTimestamp: "2026-05-25T12:04:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:03:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:04:17Z" name: switch-graph-c1cab-68466fbf77-hzqld.18b2cd6543acaddd namespace: kserve-ci-e2e-test resourceVersion: "32001" uid: 8524ad83-dd78-4f78-a2e3-51a1072885f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-36.ec2.internal source: component: kubelet host: ip-10-0-131-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c1cab-68466fbf77 namespace: kserve-ci-e2e-test resourceVersion: "24554" uid: 96778c11-e13b-4377-a5c3-d0811095e4b2 kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: 'Created pod: switch-graph-c1cab-68466fbf77-hzqld' metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab-68466fbf77.18b2cc8086ab0ff0 namespace: kserve-ci-e2e-test resourceVersion: "24558" uid: a407235f-7339-425f-84c2-a53b89f2d6f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24552" uid: 83f4f98c-a8a7-4357-8982-3b554f8b2467 kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: Scaled up replica set switch-graph-c1cab-68466fbf77 from 0 to 1 metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab.18b2cc808601a523 namespace: kserve-ci-e2e-test resourceVersion: "24555" uid: a4625a06-f9ff-4144-83be-d5dc78f30a32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24570" uid: 582f9cc6-3fae-4bb9-9da4-fecdb50b54b9 kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: InferenceGraph [switch-graph-c1cab] is Ready metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab.18b2cc808b9ef883 namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: d302c7ca-7431-41ec-8480-16a73c87ad5c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: 582f9cc6-3fae-4bb9-9da4-fecdb50b54b9 kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: 'Failed to update status for InferenceGraph "switch-graph-c1cab": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-c1cab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab.18b2cc808de632c8 namespace: kserve-ci-e2e-test resourceVersion: "24573" uid: ce7fc0dd-e98d-4314-833b-5405fa2e620b reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T11:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: 582f9cc6-3fae-4bb9-9da4-fecdb50b54b9 kind: Event lastTimestamp: "2026-05-25T11:47:35Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-c1cab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T11:47:35Z" name: switch-graph-c1cab.18b2cc808de67ee5 namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 55884660-28da-42d5-95c2-662d6b8aa920 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:47:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: 77e1ff72-5061-4ca1-a2c0-3832ebf860b4 kind: Event lastTimestamp: "2026-05-25T11:48:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:05Z" name: switch-graph-c1cab.18b2cc84048c109d namespace: kserve-ci-e2e-test resourceVersion: "24839" uid: 05a7a1d6-fb5b-4471-9fb2-8735dff8f30c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T11:47:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c1cab namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: 77e1ff72-5061-4ca1-a2c0-3832ebf860b4 kind: Event lastTimestamp: "2026-05-25T11:48:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T11:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T11:48:05Z" name: switch-graph-c1cab.18b2cc8404f372bb namespace: kserve-ci-e2e-test resourceVersion: "24840" uid: e05592d3-8a8f-4572-a1cf-8ce54463793d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32023"