--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:56:24.977770Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-a87c3-6c9b8648b7-bmf4f to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:56:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:56:24Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67f38f96154da namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: d3bea930-483e-4dd0-ab01-6cd09ea3f560 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:56:25Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67f391324a98d namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 295dbee8-54aa-482b-8d50-b53563cd7a85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a87c3} kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:25Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67f3914de5b42 namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: a77b5aff-7f3b-4e0e-a744-ccfeb89137c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a87c3} kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: 'Created container: ensemble-graph-a87c3' metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:25Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67f3918a71bce namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 3db379f2-2e4d-49c4-b977-cda151661218 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a87c3} kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: Started container ensemble-graph-a87c3 metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:25Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67f391919a8d7 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 353a09e1-ce04-427e-8e19-58ba73a60768 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a87c3} kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Stopping container ensemble-graph-a87c3 metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:39Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67fac1d61dd95 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: fdd53fb5-bf1d-4380-abfc-e98c56fb34c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:04:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a87c3} kind: Pod name: ensemble-graph-a87c3-6c9b8648b7-bmf4f namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 10f2ec7d-e239-4761-8142-35c1e124d730 kind: Event lastTimestamp: "2026-07-28T16:05:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:06Z" name: ensemble-graph-a87c3-6c9b8648b7-bmf4f.18c67facabf93269 namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: c652fcfd-38e9-46ab-a6ae-41e8e4cde5ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-a87c3-6c9b8648b7 namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: c4288060-2300-48e3-bb9f-75a3b8f2d73e kind: Event lastTimestamp: "2026-07-28T15:56:24Z" message: 'Created pod: ensemble-graph-a87c3-6c9b8648b7-bmf4f' metadata: creationTimestamp: "2026-07-28T15:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:24Z" name: ensemble-graph-a87c3-6c9b8648b7.18c67f38f8585fcf namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: fc211ed7-782d-44e4-aaa7-ef63c3e83c3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21802" uid: b138752f-2949-42a7-bae7-4696340d42fd kind: Event lastTimestamp: "2026-07-28T15:56:24Z" message: Scaled up replica set ensemble-graph-a87c3-6c9b8648b7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:24Z" name: ensemble-graph-a87c3.18c67f38f705c38d namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: 979282ee-31f5-4115-8ebf-f01e157a8eb1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: 83b088c4-dd5d-4d76-a061-a0063fd32d38 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: InferenceGraph [ensemble-graph-a87c3] is Ready metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:25Z" name: ensemble-graph-a87c3.18c67f38ff489962 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 2a6f5cb6-9ae6-4875-9035-235bf9e56b64 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: 7abbf83c-9761-4770-8304-b4e998bceb5b kind: Event lastTimestamp: "2026-07-28T15:56:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:39Z" name: ensemble-graph-a87c3.18c67f3c759210b3 namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: cc6c8e20-55b0-4aae-94aa-61ea691bee8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: 7abbf83c-9761-4770-8304-b4e998bceb5b kind: Event lastTimestamp: "2026-07-28T15:56:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:39Z" name: ensemble-graph-a87c3.18c67f3c75f9c1f8 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: dee92cd2-a960-4665-8516-9530022b50db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: 7abbf83c-9761-4770-8304-b4e998bceb5b kind: Event lastTimestamp: "2026-07-28T15:56:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:54Z" name: ensemble-graph-a87c3.18c67f3ff516431f namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: a5144d35-fded-4ef1-8043-6ca3ffefc546 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: 7abbf83c-9761-4770-8304-b4e998bceb5b kind: Event lastTimestamp: "2026-07-28T15:56:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:54Z" name: ensemble-graph-a87c3.18c67f3ff58f6f00 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: bf9b7551-8689-41eb-bd15-0d2260fa84ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:55:14.650608Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ad35d-7f5996df4c-kkpxm to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f28998df599 namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 548e86a7-c09a-417b-a703-0bad3002dc0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-ad35d-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f28a5551ed6 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 9a6bd78f-e473-4b59-a0c1-253c293f8151 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:15Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:15Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:55:15Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f28d6cf571e namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 6695e77c-5ab9-4df6-b39c-c878672050e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ad35d} kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:15Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f28d8bb28c4 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: ffc7c9f6-c25a-4da1-8f81-5b653b1294f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ad35d} kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:15Z" message: 'Created container: ensemble-graph-ad35d' metadata: creationTimestamp: "2026-07-28T15:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:15Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f28dbc41944 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: befcb38f-3045-483f-a05d-bbd9065ad6df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ad35d} kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:15Z" message: Started container ensemble-graph-ad35d metadata: creationTimestamp: "2026-07-28T15:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:15Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f28dc2f86c8 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: f02e7393-0cf0-40dd-badf-95d689c85838 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ad35d} kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:24Z" message: Stopping container ensemble-graph-ad35d metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:24Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f2af0e9df5e namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 0dabad47-f39d-481b-b8ea-48f29e6ce35e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ad35d} kind: Pod name: ensemble-graph-ad35d-7f5996df4c-kkpxm namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9faa982e-7304-4a68-ba08-d8d305c1888d kind: Event lastTimestamp: "2026-07-28T15:55:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T15:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:52Z" name: ensemble-graph-ad35d-7f5996df4c-kkpxm.18c67f2b9dcff6d2 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 7caf438f-4a10-4c1f-a5ca-90c068ea648f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ad35d-7f5996df4c namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: 4dd079bb-f25f-4c49-b96b-a9d7cb191187 kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'Created pod: ensemble-graph-ad35d-7f5996df4c-kkpxm' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d-7f5996df4c.18c67f28986376eb namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 7db81628-8200-4343-bd0a-0e3e49e3c507 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 12e021eb-05f2-40d1-9be7-837ecb6b8bc7 kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: Scaled up replica set ensemble-graph-ad35d-7f5996df4c from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d.18c67f2896801de4 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 636b7ad7-7257-42e3-8475-d0d723c2e713 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 8ab18db7-dde0-4fe5-a8d7-59e712e9aa1b kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: InferenceGraph [ensemble-graph-ad35d] is Ready metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d.18c67f28a1417882 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 458449dd-84ce-4e85-8157-97dabef3d9e2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 8ab18db7-dde0-4fe5-a8d7-59e712e9aa1b kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-ad35d": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-ad35d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d.18c67f28a63a5002 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 1da1d020-6ac5-4569-a351-3dc7a96edd06 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 8ab18db7-dde0-4fe5-a8d7-59e712e9aa1b kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-ad35d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:14Z" name: ensemble-graph-ad35d.18c67f28a63aabe0 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 6869fe2b-333a-4478-ab9a-5f2d865b62e6 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:54:44.831731Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20500" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-12202-predictor-7fdc67944d-8cndv to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:54:44Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21a8360ad3 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 4870189a-2f99-451b-8fe7-75c16277f181 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-12202-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:44Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21aed8d9d1 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 61849f61-5713-4911-8be0-0ab34df1f7b7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21e583496c namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: c7e44486-adc5-4c10-88c3-6210ede61a68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21e72b78e3 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 271e3457-8138-4669-a1e4-74e58d1556f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21ea0fa7b0 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: c6da310e-bd7b-4351-822d-f331b2e629c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21eaebcfff namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: aad98c2f-904e-45c8-9408-00926eafdd50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21eaf74533 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 9615be7d-9401-4f9d-9f18-f2972de98cc8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:46Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21edc28a9a namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 7a624989-9e3d-4c0a-8b3d-f8c30d88f0fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:54:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:46Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f21ee33502e namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 9ef2c59e-fb30-4644-8284-f03726a5645b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:55:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:23Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f2245d77435 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: 881b416a-c88b-4909-b93a-d2e63403b603 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:54Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f31fa9899db namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: eb80bdea-3682-40b2-bfed-95f55fa3a961 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:54Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f31fa991645 namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 87b517f9-4346-405d-a41e-f0026b405171 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12202-predictor-7fdc67944d-8cndv namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 8deeb7a0-ec33-48f3-9bef-23fd4db43fb0 kind: Event lastTimestamp: "2026-07-28T15:55:58Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:58Z" name: error-404-isvc-12202-predictor-7fdc67944d-8cndv.18c67f32ce174363 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 1c016b5a-31b5-4ecb-9e4f-2ea81103b15e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-12202-predictor-7fdc67944d namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 1467a395-59bb-406f-a97a-f9d622d75516 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: 'Created pod: error-404-isvc-12202-predictor-7fdc67944d-8cndv' metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:44Z" name: error-404-isvc-12202-predictor-7fdc67944d.18c67f21a6c9946b namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: b381f911-2c4b-4061-8021-7ba49e95324b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 936e58f6-95f7-4de1-964a-b0a71dcd6403 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: Scaled up replica set error-404-isvc-12202-predictor-7fdc67944d from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:44Z" name: error-404-isvc-12202-predictor.18c67f21a3210b17 namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: 4ab0c352-99d1-4da2-b1d0-d6798ed7ec10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: dcc3d62b-f180-41ab-ba4e-5c3fbbf530d2 kind: Event lastTimestamp: "2026-07-28T15:54:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:59Z" name: error-404-isvc-12202-predictor.18c67f252143273f namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: ec3ce348-f522-45f3-9e67-128fb487be74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: dcc3d62b-f180-41ab-ba4e-5c3fbbf530d2 kind: Event lastTimestamp: "2026-07-28T15:54:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:59Z" name: error-404-isvc-12202-predictor.18c67f2522283e19 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 52b650a5-0588-4e37-a9b7-339bb3546f30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: dcc3d62b-f180-41ab-ba4e-5c3fbbf530d2 kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:44Z" name: error-404-isvc-12202-predictor.18c67f28a12fbdbc namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: d64c67a4-491e-4be0-860a-67d6c44f3a55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: dcc3d62b-f180-41ab-ba4e-5c3fbbf530d2 kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:44Z" name: error-404-isvc-12202-predictor.18c67f28a252fa87 namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: d52971da-d5c6-48ff-a112-a1527e705138 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12202 namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: 9bb48c19-62e7-4818-a5b2-74e716b82ce5 kind: Event lastTimestamp: "2026-07-28T15:55:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:33Z" name: error-404-isvc-12202.18c67f21a2dbf7b3 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 7701eaae-f0be-4a11-b434-f7f58ab9fa3e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12202 namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 9bb48c19-62e7-4818-a5b2-74e716b82ce5 kind: Event lastTimestamp: "2026-07-28T15:55:33Z" message: InferenceService [error-404-isvc-12202] is Ready metadata: creationTimestamp: "2026-07-28T15:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:33Z" name: error-404-isvc-12202.18c67f2d0398d69f namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 0f89ff34-1ca3-4a22-8ada-bfa4b5e4b710 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:44:49.621708Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e9712edc26e namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: c99dd5b0-62f0-4dea-8596-56de5978eff9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e972c16f019 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: bb21084e-ae31-4ef7-9cb7-63a788e8f6ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e972d8584ed namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 7597dbe0-1565-45fd-bd37-2254779e8cb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e973014686a namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 1a7c642b-b55f-4195-9792-b0f4d552475b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e9730741f94 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 12d2ae86-b024-49a3-9ed4-33212201769a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e97307f8e47 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: ce746818-169b-4e54-8f37-89372fe11a06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e9733c59ac5 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: dfd9f171-2a6f-41c7-bdfc-af01e8192a73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e973424f1a2 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 76f93de6-b6aa-4eed-85c7-30cd76a4846e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:44:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:54:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:17Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67e9775ad2c0a namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: de3f36a9-8d85-40d5-8291-5dc22b936c72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67f1aa3be0167 namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: db8703d7-da0c-4816-a106-9fb2d35a9148 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67f1aa3be87bd namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: f63d0ef9-de3f-4b3c-b6a8-418fd8d2aa00 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: a191d9c3-8d75-496b-867f-026862812283 kind: Event lastTimestamp: "2026-07-28T15:54:17Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:17Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8.18c67f1b3e246aff namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 2cbaffaa-93a1-438a-a7d5-cb54f1ce8396 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4b277-predictor-6f9fc5d8b6 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 6000388c-5b6c-4d30-8d06-093f414116bf kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: 'Created pod: error-404-isvc-4b277-predictor-6f9fc5d8b6-5d5k8' metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-4b277-predictor-6f9fc5d8b6.18c67e97125dd21e namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: 6934b318-c29e-4044-8560-b2449a1de1c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 636780cc-17c6-43b2-a500-9a0f6da64c36 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Scaled up replica set error-404-isvc-4b277-predictor-6f9fc5d8b6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-4b277-predictor.18c67e97109c92a4 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 606c88ed-6b13-491c-8cf1-75b521102584 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: cf4c0200-d646-424e-aa71-61facf3c6ef7 kind: Event lastTimestamp: "2026-07-28T15:46:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:04Z" name: error-404-isvc-4b277-predictor.18c67e9a8f0ce762 namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 1119dd25-5d90-4bdf-adcf-499ee7b8fcb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: cf4c0200-d646-424e-aa71-61facf3c6ef7 kind: Event lastTimestamp: "2026-07-28T15:46:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:04Z" name: error-404-isvc-4b277-predictor.18c67e9a8f6e0eec namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 32d583d4-b565-4238-93eb-28b60fcc4cfe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: cf4c0200-d646-424e-aa71-61facf3c6ef7 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:19Z" name: error-404-isvc-4b277-predictor.18c67e9e0e1fcecc namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 54e66575-2328-422c-ba12-12f7ae8fdc78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: cf4c0200-d646-424e-aa71-61facf3c6ef7 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:19Z" name: error-404-isvc-4b277-predictor.18c67e9e0e8762ef namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: e46be45e-d0e4-4789-985e-0c4e2b03f4d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: a497ec9d-2c32-4486-9e87-3efd1a1a2ebc kind: Event lastTimestamp: "2026-07-28T15:45:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:37Z" name: error-404-isvc-4b277.18c67e9710adb32f namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: b5a8f5c5-9cd4-43ae-b167-9414006a6307 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: a497ec9d-2c32-4486-9e87-3efd1a1a2ebc kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-4b277": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4b277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-4b277.18c67e971156f115 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: f165aa4c-8e1c-4f63-9b12-30b20ace3c08 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: a497ec9d-2c32-4486-9e87-3efd1a1a2ebc kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4b277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-4b277.18c67e9711572686 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: 0b00aa90-9011-42ba-91df-267947ecc575 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: a497ec9d-2c32-4486-9e87-3efd1a1a2ebc kind: Event lastTimestamp: "2026-07-28T15:45:37Z" message: InferenceService [error-404-isvc-4b277] is Ready metadata: creationTimestamp: "2026-07-28T15:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:37Z" name: error-404-isvc-4b277.18c67ea232871db1 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 74b4e07a-2611-4594-9335-bb72c480aa50 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:55:55.081221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-50d42-predictor-8b88c967b-q9295 to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f320368254b namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: e32510d9-2fed-4dc2-993e-96a15ba8a277 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-50d42-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f320eaaad35 namespace: kserve-ci-e2e-test resourceVersion: "21510" uid: b33ba291-67c0-44d4-93e3-3ab165ddda09 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f32764ac995 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: f152be9f-c53c-4a72-a73b-c3a64b00fdfe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f32780fda53 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: 3c3be1b5-797f-4554-b3c6-adaf811dbd05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f327b6847a2 namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: 6391cc98-4326-4594-a408-0b4c649b02cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f327bdb1c2b namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 7b1ccb27-aa15-4056-85e0-632727806ccf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f327be5eafa namespace: kserve-ci-e2e-test resourceVersion: "21540" uid: e2346913-10fc-45e1-8122-a7333b816923 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f327e8d7fdf namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: de40c17c-52c9-4e36-b2fb-178c87450626 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f327f038ac8 namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 29fe835e-e5ab-4535-965d-912fc98c723a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T15:56:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:34Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67f32dfc23140 namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 8fa7aa5f-e7f1-465f-8ce4-2e0c10244b0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67fb32b0c2c33 namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 5b8213d5-fe25-47e7-9855-e2b39be92219 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50d42-predictor-8b88c967b-q9295 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 529d4acb-1ed1-46d2-a4cc-e89c1982008a kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-50d42-predictor-8b88c967b-q9295.18c67fb32b0cf861 namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 7fbce639-6c36-4dcf-957b-9ec1d687b9a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-50d42-predictor-8b88c967b namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: e0cabd7b-b375-4703-98b3-2b49ef8dcc79 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'Created pod: error-404-isvc-50d42-predictor-8b88c967b-q9295' metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-50d42-predictor-8b88c967b.18c67f3202b04055 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 4aa659da-7863-413e-b9a0-376aad616869 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 6a992273-2fac-410a-9e14-1a506f84f131 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: Scaled up replica set error-404-isvc-50d42-predictor-8b88c967b from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-50d42-predictor.18c67f320038e95f namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 74c5f929-c1fc-4be6-9f60-7014e077395b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 54805819-2764-45ac-81e2-e28f7796f050 kind: Event lastTimestamp: "2026-07-28T15:56:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:10Z" name: error-404-isvc-50d42-predictor.18c67f35805dac1c namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: 94593dcd-a421-4bb7-8648-8e08aaaf9319 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 54805819-2764-45ac-81e2-e28f7796f050 kind: Event lastTimestamp: "2026-07-28T15:56:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:10Z" name: error-404-isvc-50d42-predictor.18c67f35813494ef namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: 636199a2-28c2-4630-9bd2-767552f46aca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 54805819-2764-45ac-81e2-e28f7796f050 kind: Event lastTimestamp: "2026-07-28T15:57:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:57:10Z" name: error-404-isvc-50d42-predictor.18c67f390002f0ac namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 412f41cf-c2b3-4ac4-ba33-ea3deb7e3607 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:56:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 54805819-2764-45ac-81e2-e28f7796f050 kind: Event lastTimestamp: "2026-07-28T15:57:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:57:10Z" name: error-404-isvc-50d42-predictor.18c67f3900884404 namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 03a18373-8063-47cc-82d0-3f27716c9f1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50d42 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 3741235c-3a0e-4700-834c-20c169d6554e kind: Event lastTimestamp: "2026-07-28T15:56:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:44Z" name: error-404-isvc-50d42.18c67f32003f0bbc namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 54fcb250-47c2-45c0-9697-0b69877e6046 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50d42 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 3741235c-3a0e-4700-834c-20c169d6554e kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-50d42": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-50d42": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-50d42.18c67f3200f39fc4 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 30e3a8eb-163c-4321-b6bb-8c05980eefce reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50d42 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 3741235c-3a0e-4700-834c-20c169d6554e kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-50d42": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-50d42.18c67f3200f3e636 namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: 781be8bd-2da9-467a-b91e-57c2879fc1ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50d42 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 3741235c-3a0e-4700-834c-20c169d6554e kind: Event lastTimestamp: "2026-07-28T15:56:44Z" message: InferenceService [error-404-isvc-50d42] is Ready metadata: creationTimestamp: "2026-07-28T15:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:44Z" name: error-404-isvc-50d42.18c67f3d9c4d088f namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: fab42333-400e-4b59-bb29-5e5dfd82715c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:45:29.642606Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-642bb-predictor-6c6846787d-fmpcw to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:45:29Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0645c3c6c namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 6178f706-57e5-447d-b25f-4d8510cccc7d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-642bb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea06cd42943 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e854bbb7-928f-4754-9359-76c7f1162698 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0a1d2218f namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: a7f84fe4-7960-47a6-a30a-9d9a2bd0850c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0a333444d namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 0c8861ab-679b-4ced-9f8e-ed2bde143241 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0a633a067 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 50b26b8b-1e23-4273-a472-04de8847d4c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0a6a946d2 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 06d249f7-00ad-4f51-9540-b3c68194ab67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0a6b4c53e namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 7805ee4c-c518-43d3-8ae2-2e9ed63fef3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0a9948cce namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: e19c47ce-2a58-4287-a755-c997f632cfdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0aa0a5b7a namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: c97c5b0b-6cb8-4134-8d69-2a3a6ceba43b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:54:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:47Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67ea0d006722f namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: bad48f7c-7041-4b60-8ada-b53a24aa2e8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67f21b4f54c48 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: b135b891-766f-4a25-9754-8494c098865c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67f21b4f5cee1 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 5898446c-f024-42b1-8a2f-0f5a56928e29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: babee5aa-d467-4143-892e-bbd46556aa8e kind: Event lastTimestamp: "2026-07-28T15:54:47Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:47Z" name: error-404-isvc-642bb-predictor-6c6846787d-fmpcw.18c67f2244655370 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 0ef4d10a-7745-4b56-8a14-012db9655d4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-642bb-predictor-6c6846787d namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 4a3cc89e-64ef-47cf-9364-15ac3f43b4e5 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: 'Created pod: error-404-isvc-642bb-predictor-6c6846787d-fmpcw' metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:29Z" name: error-404-isvc-642bb-predictor-6c6846787d.18c67ea063f1275a namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: b8537562-b642-43f6-9aeb-a20984a75a5c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: 1d73c943-8942-4251-a9c7-f33b87dc9d62 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Scaled up replica set error-404-isvc-642bb-predictor-6c6846787d from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:29Z" name: error-404-isvc-642bb-predictor.18c67ea0624d416c namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 62054a39-7633-4974-95f4-a4206159a20c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: a942d965-90cd-494d-9434-c573bfbc1238 kind: Event lastTimestamp: "2026-07-28T15:45:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:44Z" name: error-404-isvc-642bb-predictor.18c67ea3e0d5df62 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 2a81e98f-ba35-4920-ac1d-0e19faeff8b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: a942d965-90cd-494d-9434-c573bfbc1238 kind: Event lastTimestamp: "2026-07-28T15:45:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:44Z" name: error-404-isvc-642bb-predictor.18c67ea3e1489852 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 5c50d75c-34f5-4afa-8b21-eab6711fe43b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:45:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: a942d965-90cd-494d-9434-c573bfbc1238 kind: Event lastTimestamp: "2026-07-28T15:46:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:45:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:44Z" name: error-404-isvc-642bb-predictor.18c67ea75ff041e5 namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 8ec6b145-78a6-4367-ba83-6e0f60146f73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:45:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: a942d965-90cd-494d-9434-c573bfbc1238 kind: Event lastTimestamp: "2026-07-28T15:46:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:45:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:44Z" name: error-404-isvc-642bb-predictor.18c67ea7605ddfda namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: d2bfd5bc-b310-4ae4-b719-cceb5aa2ab04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-642bb namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 3a8619d3-d2be-40ff-8ccb-394aaeafd8a0 kind: Event lastTimestamp: "2026-07-28T15:46:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:46:17Z" name: error-404-isvc-642bb.18c67ea062688dee namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 353e8df7-0c6a-4d44-88a7-b69ee27acab2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-642bb namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 3a8619d3-d2be-40ff-8ccb-394aaeafd8a0 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-642bb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-642bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:29Z" name: error-404-isvc-642bb.18c67ea062ff288f namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 0d07f295-06a8-4e67-82bd-315827238341 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-642bb namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 3a8619d3-d2be-40ff-8ccb-394aaeafd8a0 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-642bb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:29Z" name: error-404-isvc-642bb.18c67ea062ff66a1 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: dd5526da-e299-4a84-8ccf-1f6236539fb0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-642bb namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: 3a8619d3-d2be-40ff-8ccb-394aaeafd8a0 kind: Event lastTimestamp: "2026-07-28T15:46:17Z" message: InferenceService [error-404-isvc-642bb] is Ready metadata: creationTimestamp: "2026-07-28T15:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:46:17Z" name: error-404-isvc-642bb.18c67eab8d270a7c namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 6c2739dc-1d21-4f4f-9e20-bfdf00f93e60 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:43:09.566647Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:43:09Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e7fc72ec6d0 namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: dc5f7ca9-d420-410c-b2eb-d05150651b01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8fc0c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:09Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e7fd4be588a namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 4ac69504-c5a2-4cf5-a593-1a90ef5911db reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:43:10Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e80046ebc7e namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 158615e5-2a54-43df-8b1c-22a28f30f3d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e8005dbf769 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: de32d834-19bf-403f-9f75-54b3e7bf0f8a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" in 12.989s (12.989s including waiting). Image size: 1333945965 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e830c12bc72 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d2043a8b-4046-429b-b2ee-4e80b1d6aa33 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e8310761aa9 namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: d67ac15f-ab80-41d6-bfc3-64dcd290a983 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e8310f085cf namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 5a574d1a-eec1-4e3f-8924-4525c147d622 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e8310fe6435 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: e2af0ee0-5f2a-4035-ae5c-6e5b71538f10 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.285s (2.285s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:25Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e8399351ad2 namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 0598fca7-6c84-402e-8640-f6595115b7af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:26Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e839c6405c4 namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 769f4a48-c1dc-4aae-981f-ef7f35d0a7e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:43:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:26Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e839cd54568 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: f718629a-8f4c-45f6-9c54-c62b54796bf5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:44:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:02Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e83d4383a97 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 9da01880-1af5-4e91-afe2-3ce05e7ce1e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e97172d7098 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: abcb800a-73aa-4cb3-a700-b6c455a81c86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 1b9a48a5-6386-453b-b01a-2f6de330d82f kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:49Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp.18c67e97172e036c namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 9f90c386-ddd5-4056-8653-d59f4fedd7b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8fc0c-predictor-548ddb8fdb namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: e6e21551-b95c-42d5-9e67-c9736f149fe4 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'Created pod: error-404-isvc-8fc0c-predictor-548ddb8fdb-56zdp' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: error-404-isvc-8fc0c-predictor-548ddb8fdb.18c67e7fc6b7ce95 namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: 77be9666-3bc6-476b-9fe4-64299b3fc2a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: 906ba505-27b2-4dab-a04d-2870cc5bd0d6 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: Scaled up replica set error-404-isvc-8fc0c-predictor-548ddb8fdb from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: error-404-isvc-8fc0c-predictor.18c67e7fc51fe117 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 2d0ab6d9-a351-4865-b39e-7d34e434f7d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 5b805934-938c-4cb4-a852-d865173b1555 kind: Event lastTimestamp: "2026-07-28T15:43:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:39Z" name: error-404-isvc-8fc0c-predictor.18c67e8343946a2f namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 1721d5cb-0316-464a-b3be-315753acbc88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 5b805934-938c-4cb4-a852-d865173b1555 kind: Event lastTimestamp: "2026-07-28T15:43:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:39Z" name: error-404-isvc-8fc0c-predictor.18c67e8343fe911b namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 91c085f6-89c6-4ee5-8c63-3e5714dcdeac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 5b805934-938c-4cb4-a852-d865173b1555 kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:39Z" name: error-404-isvc-8fc0c-predictor.18c67e8a415bbb47 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: c337d1ff-6559-4e6e-ae27-3e9812fcc0c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 5b805934-938c-4cb4-a852-d865173b1555 kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:39Z" name: error-404-isvc-8fc0c-predictor.18c67e8a41c38390 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: b6185140-9c02-4d18-abd1-eb15d84b2fdc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8fc0c namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 5db0af2a-1acd-4a51-bf6e-834cbdf096c5 kind: Event lastTimestamp: "2026-07-28T15:44:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:13Z" name: error-404-isvc-8fc0c.18c67e7fc531a487 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 5fbcf296-4743-454b-b1fe-6e0341a81197 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8fc0c namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 5db0af2a-1acd-4a51-bf6e-834cbdf096c5 kind: Event lastTimestamp: "2026-07-28T15:44:13Z" message: InferenceService [error-404-isvc-8fc0c] is Ready metadata: creationTimestamp: "2026-07-28T15:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:13Z" name: error-404-isvc-8fc0c.18c67e8e90389446 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: dacbb972-5f3c-4ce6-bf45-6e7056a48ecb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:55:25.197621Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a87c3-predictor-578565c866-hnjq7 to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b0e349058 namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: b342ec24-508f-4910-bb06-37b8c4dc64b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b28e57883 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 13d60085-b351-4cf1-92e4-51e274cd9076 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b2ace1efa namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: fd63f82f-b2c5-4beb-822e-cca8acba6a5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b2db01943 namespace: kserve-ci-e2e-test resourceVersion: "21070" uid: 4d9ea720-d276-443c-9347-9c623524e336 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b2e237a5e namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: d5b58519-e54b-4bd0-8cff-8d7a51445d66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b2e2dc83b namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: cc69afde-20f7-467e-b25a-8f20b7f16726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b311e0b2f namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 489c384e-0d70-4c8f-9c36-304c9e8fa9d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2b318caa21 namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: 7685f8cc-a94e-4a53-ba58-fc1bab1714b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T15:56:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:03Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67f2ba03044f2 namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: 998a2c74-4413-4cd8-a094-3c832e637eab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67fac3311e111 namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: b0b2737b-78ac-4629-870b-4efab2f9c5bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a87c3-predictor-578565c866-hnjq7 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: de2b6fe3-5b72-497a-860e-a7a19ecbcfa4 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-a87c3-predictor-578565c866-hnjq7.18c67fac331230d0 namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 88c0df55-24a1-4eec-93d9-bf649b455322 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a87c3-predictor-578565c866 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 885a3b53-21b6-4d1d-8281-abb6d1155ef4 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: 'Created pod: error-404-isvc-a87c3-predictor-578565c866-hnjq7' metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor-578565c866.18c67f2b0d812971 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: cafd49c3-ea7c-4286-9e47-2a44325950cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: eac3c1dc-a5a3-4550-83bd-f00546d33ae7 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Scaled up replica set error-404-isvc-a87c3-predictor-578565c866 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3-predictor.18c67f2b0acf92c6 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 5f1e0ad2-76a4-49f4-bda2-728c40a55787 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: b941cfd5-09c1-4d50-8c9f-f2f367e4e963 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-a87c3-predictor.18c67f2e8948d7e8 namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 10cd71fd-2671-47d3-907e-00efeb3fc986 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: b941cfd5-09c1-4d50-8c9f-f2f367e4e963 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:55Z" name: error-404-isvc-a87c3-predictor.18c67f2e8a020dde namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: a3fb1dde-7d23-4466-b729-77773164ddc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:56:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: b941cfd5-09c1-4d50-8c9f-f2f367e4e963 kind: Event lastTimestamp: "2026-07-28T15:56:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:40Z" name: error-404-isvc-a87c3-predictor.18c67f3587ff79f1 namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 3ccf0be9-02a3-4a63-aa9c-c2b83fde3a50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:56:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: b941cfd5-09c1-4d50-8c9f-f2f367e4e963 kind: Event lastTimestamp: "2026-07-28T15:56:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:40Z" name: error-404-isvc-a87c3-predictor.18c67f3588857edd namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 78f11ebc-81cf-467f-a815-b6447efc8136 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 99648f5f-5ee1-4dc3-a9d7-f5bde39bcf5e kind: Event lastTimestamp: "2026-07-28T15:56:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:13Z" name: error-404-isvc-a87c3.18c67f2b0ad75752 namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: f17f8861-d276-476f-8fa4-55ff0bed2043 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 99648f5f-5ee1-4dc3-a9d7-f5bde39bcf5e kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-a87c3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a87c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3.18c67f2b0c114e95 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 29c3585f-5c84-4d38-b349-fb1be8c0b613 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 99648f5f-5ee1-4dc3-a9d7-f5bde39bcf5e kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a87c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:25Z" name: error-404-isvc-a87c3.18c67f2b0c11a46e namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 344a489a-c56a-43e0-8f82-7e8307e0c075 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 99648f5f-5ee1-4dc3-a9d7-f5bde39bcf5e kind: Event lastTimestamp: "2026-07-28T15:56:13Z" message: InferenceService [error-404-isvc-a87c3] is Ready metadata: creationTimestamp: "2026-07-28T15:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:13Z" name: error-404-isvc-a87c3.18c67f365cfa26fc namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: e40992a7-ec93-46d7-8527-928946f797fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:54:14.612460Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ad35d-predictor-6cb9647b76-schlh to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1a9f009785 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 5ff042d2-476a-49ac-be85-999dac5a219d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1ab8813336 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: b07abc15-a09e-4767-a2c8-e0aa32067bbd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1abab7fb7e namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: e7533ada-ffd5-451a-86ab-00b4d1b4b6c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1abd753456 namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 5ec5cd12-e659-4230-9d2b-ac084d27ab16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1abde56dc5 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: bd014ca5-6950-4ba3-8341-df51ba299329 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1abdedfc6f namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: f07bf8b7-6404-49f6-a35b-369ce8d33cb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1ac1550811 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 4bd5422b-1d5f-4c9e-95dd-cd7a53bfc1ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1ac1da8771 namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: 8ff3741d-0b7d-454d-989f-635ca27f91bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:54:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:52Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f1b06de6af7 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 94dff56b-9be1-411f-91bb-4e1c5e5bcbfa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:55:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:24Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f2afaf4b93d namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 1bb9943c-0a3b-4cde-bce5-6f2625ace2bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:55:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:24Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f2afaf53b06 namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 44adbf6b-bf34-4562-8067-31837ff1baf4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 1d5cb8ed-0736-43af-b3b4-07b38ab90dc2 kind: Event lastTimestamp: "2026-07-28T15:55:27Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:27Z" name: error-404-isvc-ad35d-predictor-6cb9647b76-schlh.18c67f2b8f00ca95 namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 5688124c-9f5e-4ece-a1a8-266c25db4f79 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ad35d-predictor-6cb9647b76 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 17511fa7-df12-4b15-90ee-ba513e37dd7a kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: 'Created pod: error-404-isvc-ad35d-predictor-6cb9647b76-schlh' metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-ad35d-predictor-6cb9647b76.18c67f1a9e741777 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 9c1734d3-8ec4-4ecc-93f6-5aa83cf3c42d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 98015b37-8b20-4614-9647-dec652caeae5 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Scaled up replica set error-404-isvc-ad35d-predictor-6cb9647b76 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-ad35d-predictor.18c67f1a9c82f9ef namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 7854a779-f566-4fb9-b0d7-20a112271c16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: cc5ea2f1-e0bb-46f3-9984-2be6eed617e9 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:44Z" name: error-404-isvc-ad35d-predictor.18c67f1e1aee08de namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: c27d91e7-8698-4546-ae36-4e4715c28711 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: cc5ea2f1-e0bb-46f3-9984-2be6eed617e9 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:44Z" name: error-404-isvc-ad35d-predictor.18c67f1e1b6c4f39 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 77d23f04-9e23-484c-9cfc-45b2dc3ba587 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: cc5ea2f1-e0bb-46f3-9984-2be6eed617e9 kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:14Z" name: error-404-isvc-ad35d-predictor.18c67f251aa58c46 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: d7311c01-96b1-438d-a74e-956330393594 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: cc5ea2f1-e0bb-46f3-9984-2be6eed617e9 kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:14Z" name: error-404-isvc-ad35d-predictor.18c67f251b9ca112 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: db930458-fdf2-4552-afbd-55f76a5eb86d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: eec44933-0d53-43c1-a935-38493bec515b kind: Event lastTimestamp: "2026-07-28T15:55:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:02Z" name: error-404-isvc-ad35d.18c67f1a9c7f767c namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: d54b4149-5af3-4b25-b456-5bb48805106d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: eec44933-0d53-43c1-a935-38493bec515b kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-ad35d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ad35d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-ad35d.18c67f1a9d1038e9 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: 75224097-4305-4382-a1b1-0ea74b3f79f2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: eec44933-0d53-43c1-a935-38493bec515b kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ad35d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:54:14Z" name: error-404-isvc-ad35d.18c67f1a9d1091c1 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 185290a7-9ab9-4df9-b9e3-2061f8490a0f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: eec44933-0d53-43c1-a935-38493bec515b kind: Event lastTimestamp: "2026-07-28T15:55:02Z" message: InferenceService [error-404-isvc-ad35d] is Ready metadata: creationTimestamp: "2026-07-28T15:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:02Z" name: error-404-isvc-ad35d.18c67f25cca32a8f namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 1d04e599-e564-434b-bf1f-e15c5bd7fb39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:04:39.810188Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac2fb0cc75 namespace: kserve-ci-e2e-test resourceVersion: "25677" uid: bf727bf7-eb40-44b6-8037-d6cf60b130ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac49757274 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: a33f652f-79b5-4c10-92d8-f1b2ff1985d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac4ace3340 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: fdc3ea5e-c822-4df9-aaa1-58d9ea222304 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac4dee1be1 namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: ab136c94-dde0-4c8e-b834-f73bd5d5f9fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac4e62f4d0 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 7fad389d-c277-4b05-8eac-4f8dd05d6446 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac4e6c9b5b namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: ba6a4aa2-f657-4624-8793-e9607a9d5775 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac51a82206 namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 63dc759f-4969-4644-aae8-08c7d8fe43eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fac52192ff6 namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: ea293952-5cac-472b-ae6d-8684c2d1b569 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:04:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:05:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:17Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67faca08de8df namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: 0597021e-6d4e-422d-9fef-6afcbec5ba33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fbc896aecf4 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 2823664b-7cea-4427-bebd-2de28e3be3cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fbc896b85f8 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: cfb9ee49-4fdb-43be-8754-9c370e76d26e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 95a1bb7a-5770-4953-abb6-d23aa5ecf273 kind: Event lastTimestamp: "2026-07-28T16:05:52Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:52Z" name: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp.18c67fbd28a994fb namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: b04dc948-d889-4f7c-8a5a-a829adf081dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b87a7-predictor-cc4c5464d namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 9ccb5914-35af-4850-b40f-824bf7dfc3b4 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'Created pod: error-404-isvc-b87a7-predictor-cc4c5464d-rvlzp' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-b87a7-predictor-cc4c5464d.18c67fac2f4d7ba0 namespace: kserve-ci-e2e-test resourceVersion: "25675" uid: c21562f3-ba63-4434-a941-6c21e43cd187 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: 1991bbe0-7a79-4285-b6f1-757e43f20d87 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Scaled up replica set error-404-isvc-b87a7-predictor-cc4c5464d from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-b87a7-predictor.18c67fac2d723fc9 namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: f433178e-b2dd-4e4d-b9ca-994e1120ea85 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:04:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: e6ecd9b3-e8d8-43c0-8d7c-1df79f3d0a02 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-b87a7-predictor.18c67fafac0c2f9d namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 9b7ffd14-c35a-4f7c-bd10-52dc02298092 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:04:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: e6ecd9b3-e8d8-43c0-8d7c-1df79f3d0a02 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-b87a7-predictor.18c67fafac75d8be namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: e7651123-d6eb-417b-8b13-7d5fb3307d13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: e6ecd9b3-e8d8-43c0-8d7c-1df79f3d0a02 kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: error-404-isvc-b87a7-predictor.18c67fb6aa01190f namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: 0fb4a959-6a3a-4440-8eeb-8f20d6ab4b2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: e6ecd9b3-e8d8-43c0-8d7c-1df79f3d0a02 kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: error-404-isvc-b87a7-predictor.18c67fb6aa6b2cb9 namespace: kserve-ci-e2e-test resourceVersion: "26405" uid: 66d722dd-49bb-41a3-a85d-ea067c4bb8fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 4c7a8ea3-6d68-4a0a-ae16-ebf33afc4e9e kind: Event lastTimestamp: "2026-07-28T16:05:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:27Z" name: error-404-isvc-b87a7.18c67fac2d872757 namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: baea9740-c8d2-41d2-9dce-1a0d1daf61c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 4c7a8ea3-6d68-4a0a-ae16-ebf33afc4e9e kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-b87a7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b87a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-b87a7.18c67fac2e0390a7 namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: 7f84a13f-dbc7-4198-8982-600d1cf10e14 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 4c7a8ea3-6d68-4a0a-ae16-ebf33afc4e9e kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b87a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:04:39Z" name: error-404-isvc-b87a7.18c67fac2e042e93 namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: 09a345df-afe5-4ab0-84d6-5387f2485c17 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 4c7a8ea3-6d68-4a0a-ae16-ebf33afc4e9e kind: Event lastTimestamp: "2026-07-28T16:05:27Z" message: InferenceService [error-404-isvc-b87a7] is Ready metadata: creationTimestamp: "2026-07-28T16:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:27Z" name: error-404-isvc-b87a7.18c67fb75c642331 namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 25674fb6-4ff1-41e1-a4b2-0f9d5f5e3173 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:05:09.909507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb331bff65f namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: 8bd4dfa4-3086-4705-bf04-0e92419d8410 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb34bcb7a72 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 635f906b-df40-40fe-960c-4b138c2d0905 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb34d448867 namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 80887789-7cfe-47a4-8897-d18097a34352 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb3500c4900 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 49e34a2d-b3da-43e4-b93b-296a3b3e01b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb35086cac9 namespace: kserve-ci-e2e-test resourceVersion: "26093" uid: f031e92f-a9a9-4dff-8deb-1b5bf04fa84f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb350928746 namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 6c9550c4-d7a8-4e9c-8f74-c994e61851d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb35372fcde namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: dcb071b3-48ae-4b6d-b392-7116a0c011a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb353f944c5 namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 8afc2792-42b3-409e-ae59-60e9f71858cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:05:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:47Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c67fb3a4259b35 namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: bd2c336f-fcb2-457e-9fa9-a9ddf99c34ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:22:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:29Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c680a53e0c7069 namespace: kserve-ci-e2e-test resourceVersion: "33745" uid: ccab8d69-a066-40a7-b491-379c7ea52bfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 8a666dfa-d384-4978-9207-ab37bf0c89a9 kind: Event lastTimestamp: "2026-07-28T16:22:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:29Z" name: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6.18c680a53e0d0934 namespace: kserve-ci-e2e-test resourceVersion: "33746" uid: 2ef4290f-c0f3-4da8-92b8-9ba43fe290f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bd785-predictor-755d7cb5cd namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: e6b10847-566b-4b19-8bd6-a6e9a169a360 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'Created pod: error-404-isvc-bd785-predictor-755d7cb5cd-cd5b6' metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-bd785-predictor-755d7cb5cd.18c67fb33125a96b namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: ac2d6b4e-b17d-43d5-bf0b-2e91444505ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: 9f9304c8-715d-42b3-8c1b-c90883a17e7d kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Scaled up replica set error-404-isvc-bd785-predictor-755d7cb5cd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-bd785-predictor.18c67fb32f798697 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 58caab81-a908-4b57-8f8e-59e471d04bfd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: ef9240c8-623b-4a71-8895-31c82ae6f275 kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: error-404-isvc-bd785-predictor.18c67fb6ae187bf9 namespace: kserve-ci-e2e-test resourceVersion: "26412" uid: 0e67369f-8af1-4145-aeed-4ebb2d1ef34c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: ef9240c8-623b-4a71-8895-31c82ae6f275 kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: error-404-isvc-bd785-predictor.18c67fb6ae60bc44 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: 2a79ebf3-d17e-4dfe-9b70-53a63b9db200 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: ef9240c8-623b-4a71-8895-31c82ae6f275 kind: Event lastTimestamp: "2026-07-28T16:06:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:24Z" name: error-404-isvc-bd785-predictor.18c67fbdabd717a1 namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 38b080b0-14f9-4b1a-81aa-2e1f41b1ab19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: ef9240c8-623b-4a71-8895-31c82ae6f275 kind: Event lastTimestamp: "2026-07-28T16:06:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:24Z" name: error-404-isvc-bd785-predictor.18c67fbdac40031f namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: e33ba952-f1de-4dab-ae86-7caca2299761 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: bb9ceff8-907d-43e8-af8d-a5d467588205 kind: Event lastTimestamp: "2026-07-28T16:05:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:57Z" name: error-404-isvc-bd785.18c67fb32fab5b7a namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 96ee9add-59be-4d8f-82c5-3d722ec6dba8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: bb9ceff8-907d-43e8-af8d-a5d467588205 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-bd785": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bd785": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-bd785.18c67fb330310f0c namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: f592758f-35f7-4f4d-8e2c-06654ac829ce reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: bb9ceff8-907d-43e8-af8d-a5d467588205 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bd785": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:09Z" name: error-404-isvc-bd785.18c67fb330315ec8 namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 5eee7a2a-fcb4-410a-891a-ae3869919462 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: bb9ceff8-907d-43e8-af8d-a5d467588205 kind: Event lastTimestamp: "2026-07-28T16:05:57Z" message: InferenceService [error-404-isvc-bd785] is Ready metadata: creationTimestamp: "2026-07-28T16:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:57Z" name: error-404-isvc-bd785.18c67fbe5ff56052 namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: 0dff0016-e856-4931-a55b-7df6301c4a30 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:05:50.167470Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dd6ec-predictor-d89d98c86-sswbf to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbc914fb616 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 3a705e5a-8335-48eb-86b9-3d5063b57864 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcaab88915 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: dc7288fa-93c6-4bec-96fa-ecf7c7851e5a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcac3f1546 namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 5d71f004-6dc0-4e59-bf46-280bed9a508b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcaf008bb9 namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: d37b6abb-37be-4e6b-8c0a-8951f84138ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcaf733f4b namespace: kserve-ci-e2e-test resourceVersion: "26619" uid: 30ce578c-fbb3-476a-be57-8468cd657f7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcaf7d32fb namespace: kserve-ci-e2e-test resourceVersion: "26620" uid: 09a74302-7366-4aff-8006-92d952e33cd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcb3981fad namespace: kserve-ci-e2e-test resourceVersion: "26621" uid: a8807ae4-c40f-4ece-87f6-8cdec9451230 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcb40ce015 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 539f5665-efc2-48c3-b32b-942cffe00834 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:15:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:07Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c67fbcc291dcfc namespace: kserve-ci-e2e-test resourceVersion: "30692" uid: 5451815f-0a1e-4e5e-9eb7-3835025e3068 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:15:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:04Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c6803db8e93392 namespace: kserve-ci-e2e-test resourceVersion: "30674" uid: 7985ed0f-22c2-4f5d-a43f-6a8778cfec1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:15:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:04Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c6803db8e9a67f namespace: kserve-ci-e2e-test resourceVersion: "30675" uid: f7e79ba6-68c1-4a99-9c9c-1b6d29a4b3c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: e50225dc-6162-45f4-aa5e-d37676c111e8 kind: Event lastTimestamp: "2026-07-28T16:15:06Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:06Z" name: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf.18c6803e36f536d6 namespace: kserve-ci-e2e-test resourceVersion: "30691" uid: ead8d9ed-9ae2-4ba5-ad26-7b17abd71c5f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dd6ec-predictor-d89d98c86 namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: f21fa949-0ea3-4071-98b6-9df44901d7c5 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'Created pod: error-404-isvc-dd6ec-predictor-d89d98c86-sswbf' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor-d89d98c86.18c67fbc90ac8742 namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: 1c028b06-98ab-4cda-a791-f2eec6529b55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: 7ae2ef9c-61de-4184-bfde-5bbe04880148 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Scaled up replica set error-404-isvc-dd6ec-predictor-d89d98c86 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec-predictor.18c67fbc8ef271bb namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: 79ab72d1-e555-4056-9697-538cd938d362 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: e77f441b-40dc-4aef-8b12-b032c97b46f2 kind: Event lastTimestamp: "2026-07-28T16:07:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:07:05Z" name: error-404-isvc-dd6ec-predictor.18c67fc00d73d5a1 namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: a22d8059-4e84-4b91-ad12-d7cf6fd20ae7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: e77f441b-40dc-4aef-8b12-b032c97b46f2 kind: Event lastTimestamp: "2026-07-28T16:07:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:07:05Z" name: error-404-isvc-dd6ec-predictor.18c67fc00dd40b17 namespace: kserve-ci-e2e-test resourceVersion: "27337" uid: 9c2dc4de-58f0-434b-9c5b-30393d097955 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: e77f441b-40dc-4aef-8b12-b032c97b46f2 kind: Event lastTimestamp: "2026-07-28T16:06:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:20Z" name: error-404-isvc-dd6ec-predictor.18c67fc38c855578 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 6d164bf7-e90f-47f0-9fd9-958411a616ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: e77f441b-40dc-4aef-8b12-b032c97b46f2 kind: Event lastTimestamp: "2026-07-28T16:06:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:20Z" name: error-404-isvc-dd6ec-predictor.18c67fc38d08dbad namespace: kserve-ci-e2e-test resourceVersion: "26945" uid: fae5f165-9647-4396-a543-d10034128c20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 6e4e7385-e57d-490a-ac98-41857b1f721f kind: Event lastTimestamp: "2026-07-28T16:06:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:06:37Z" name: error-404-isvc-dd6ec.18c67fbc8f10dcd3 namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 907d4fee-a872-4cc9-9709-f35d3d83f9ae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 6e4e7385-e57d-490a-ac98-41857b1f721f kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-dd6ec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dd6ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec.18c67fbc8f866964 namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 23af9e38-dca6-4c5f-9898-be1022bc5d57 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 6e4e7385-e57d-490a-ac98-41857b1f721f kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dd6ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:50Z" name: error-404-isvc-dd6ec.18c67fbc8f86d011 namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: 64612d5e-598a-417b-b879-842e98063292 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "27077" uid: 6e4e7385-e57d-490a-ac98-41857b1f721f kind: Event lastTimestamp: "2026-07-28T16:06:37Z" message: InferenceService [error-404-isvc-dd6ec] is Ready metadata: creationTimestamp: "2026-07-28T16:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:06:37Z" name: error-404-isvc-dd6ec.18c67fc77e68f123 namespace: kserve-ci-e2e-test resourceVersion: "27078" uid: 5c6d0a76-a095-4a18-ac5a-1a8e26ca1955 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:43:09.468404Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e7fc153ba33 namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: ee6fca4c-267c-444c-a10f-03be80a3548a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e7fd29efa6f namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: a678b6b6-d310-486b-af6b-e69f696d0f4b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:43:10Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e7fffb08cba namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: f2205312-e3c1-46d5-8edb-d285d343d950 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:06e49304d7aff24f41f8f0b7d4ea2dcf886a7d2132c6967087fee7fad06051a4" already present on machine metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e80014cd47b namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 28296b4d-c5d9-49f4-a71f-9430d5f69d01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e8004fa878a namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 2b8aa954-3dfb-42b9-a90d-32ec910b27c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e80055d7ee9 namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: f05ffd93-b5ac-4fff-b0ca-718446d298dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:16Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T15:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:16Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e814dc23622 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 6b37909d-9784-492c-9af8-1ec48447f695 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 20.346s (20.346s including waiting). Image size: 1555662260 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e860a86c4cc namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: 53e95f3f-6253-4246-8c90-58a1df0b1e8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e86128d6168 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 2ad4edc2-15a4-4661-a705-797f3d059952 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e8613f95429 namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: d0ded2ef-3603-4409-903c-f91259f50444 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e8614070579 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: eaa0416c-3875-4361-b11e-43e47c6ee2a2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.643s (3.643s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e86ed2fccde namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 1a5e6606-48a2-4786-9540-a4266bbf1e54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e86f1c73ce1 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 83e5536a-62ff-4ebd-a717-dce12a77a49b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e86f28885b6 namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: de46f576-4f4b-4c19-8c1d-e5da8db2f0a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T15:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:45:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:32Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67e876611ad68 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 12569855-4d8d-40ef-9ba3-79c350b6489e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67ea05bff6bcf namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 04786d2c-8e2d-4144-9e1c-71db132c4765 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67ea05c000829 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: fccdb82a-eb52-4239-a246-1cdf9315736a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 8e64bee5-715e-428a-8a99-33e462d48b18 kind: Event lastTimestamp: "2026-07-28T15:45:33Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:33Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp.18c67ea13e78a61a namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: def3e585-d08b-4685-a3da-b152ad254b8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6d55f5d589 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 2d992f60-463b-4e70-971b-de1f784701d6 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6d55f5d589-6knsp' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589.18c67e7fc0e2bf01 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 4899ce28-16bb-41f3-9c66-f1f0d81e0b5f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 2252517d-05b2-4739-ad0f-3bce26687b18 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6d55f5d589 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-1-predictor.18c67e7fbdf1f5b9 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 6a8937e5-6c56-407a-8437-1d2c55b00232 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: ab3628ca-486f-4cdd-892b-cb27a7aaa9e2 kind: Event lastTimestamp: "2026-07-28T15:44:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:09Z" name: isvc-sklearn-graph-1-predictor.18c67e833c7a6b9b namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: f7f3e2b1-9eb4-446f-88dc-aad06575ed8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: ab3628ca-486f-4cdd-892b-cb27a7aaa9e2 kind: Event lastTimestamp: "2026-07-28T15:44:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:09Z" name: isvc-sklearn-graph-1-predictor.18c67e833ce874a6 namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: 93b83487-c35f-4af8-83ff-0d60916ad8c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: ab3628ca-486f-4cdd-892b-cb27a7aaa9e2 kind: Event lastTimestamp: "2026-07-28T15:45:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:24Z" name: isvc-sklearn-graph-1-predictor.18c67e9137b66c3e namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: f5eea050-e620-4c38-9e7d-5831a08320aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: ab3628ca-486f-4cdd-892b-cb27a7aaa9e2 kind: Event lastTimestamp: "2026-07-28T15:45:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:24Z" name: isvc-sklearn-graph-1-predictor.18c67e9138233fa5 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: 106422a9-daca-45d3-b798-9e12c048ea66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: dc30b562-06d3-4507-9b07-7556a94e0b1c kind: Event lastTimestamp: "2026-07-28T15:44:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:52Z" name: isvc-sklearn-graph-1.18c67e7fbe2f3894 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: ee9a79b9-6434-4734-b52d-ea52b25ee0ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: dc30b562-06d3-4507-9b07-7556a94e0b1c kind: Event lastTimestamp: "2026-07-28T15:44:52Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T15:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:52Z" name: isvc-sklearn-graph-1.18c67e97b652e54f namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 6a3ed96c-9c5f-4189-b065-1edd33da530f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:43:09.893165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e7fdaa4f8e9 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: d6b7aab5-47dd-475a-8351-16a24b324f0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:43:10Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e7ff494567f namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 5f8588b8-29f4-4af8-8d4e-5c4bb34cfd7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:06e49304d7aff24f41f8f0b7d4ea2dcf886a7d2132c6967087fee7fad06051a4" metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e7ff61366d0 namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 704159e6-c5db-433f-852e-9874a1e763b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:06e49304d7aff24f41f8f0b7d4ea2dcf886a7d2132c6967087fee7fad06051a4" in 13.162s (13.162s including waiting). Image size: 300978647 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e8306a341fb namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: d2b340cd-7bea-4aa2-bc70-535d53c6e06e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e83098e1c0a namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 6d17cb92-6fe6-48fb-b024-28b7ca0cc296 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T15:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:23Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e830a00e514 namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: 7b865f6a-0be0-427f-8ba8-985a4709d931 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:27Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T15:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:27Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e84101c4341 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 489d9a95-d97d-41ec-81b6-678cd6819026 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:33Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 5.562s (5.562s including waiting). Image size: 1555662260 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:33Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e855bafc0d0 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: d219aaad-4735-4e5d-b68a-d781ec504f8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:33Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e855f3f3a6f namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: d0c6ba79-4a31-4df6-8f2a-803d1c4938ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:33Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e855fb3a4e5 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 6ab4d731-714a-4193-bb0c-378cbbe88d69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:33Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e855fc88b22 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 42525e08-f1db-4d35-8ab4-5a1a150dcce2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:33Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e8562d05b5a namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: 95f596e7-7719-457c-9db6-dcd256488619 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:43:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:33Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e8563483767 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 62876b86-638a-4908-9f1f-e8b60628896a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T15:43:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:45:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67e85b33362ea namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: bc0fbc41-05b8-4d8f-852f-1a7b9848dd11 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67ea0587641a9 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: d6e56b73-c011-46de-ac23-a09e90829ac2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67ea05876a099 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: fc5f5b66-8b04-45e0-9d03-a3ebef74ccd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: cb8f4c23-7816-4b84-a079-74ae156a87a6 kind: Event lastTimestamp: "2026-07-28T15:45:31Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:31Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k.18c67ea0b58fe93c namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 44fd3c24-6df1-49b6-b3d9-329df4729188 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7d44765f95 namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: e276495f-a12c-4b7a-803d-f1173be9ba6c kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7d44765f95-j7z2k' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-2-predictor-7d44765f95.18c67e7fda0c715d namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 02c11798-69da-4a53-94a4-99ada1e0fb2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 7b04d10e-4f9c-4098-b90e-f90d3ec1d8fc kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7d44765f95 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: isvc-sklearn-graph-2-predictor.18c67e7fd6bcacf9 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 657c0af1-a8b1-4957-94ad-5e0cd9a1e311 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: d73f1d35-2c31-4444-9bff-5465921e98f8 kind: Event lastTimestamp: "2026-07-28T15:43:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:39Z" name: isvc-sklearn-graph-2-predictor.18c67e835530f507 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 63d53135-acb0-4f2d-9737-a2e898a82740 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: d73f1d35-2c31-4444-9bff-5465921e98f8 kind: Event lastTimestamp: "2026-07-28T15:43:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:39Z" name: isvc-sklearn-graph-2-predictor.18c67e8355c5ea07 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 53112da7-63ae-4d3a-ba71-dbead76c5e6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: d73f1d35-2c31-4444-9bff-5465921e98f8 kind: Event lastTimestamp: "2026-07-28T15:45:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:24Z" name: isvc-sklearn-graph-2-predictor.18c67e8a531e2b78 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 951aebbd-b015-4a2d-a276-894326080464 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:43:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: d73f1d35-2c31-4444-9bff-5465921e98f8 kind: Event lastTimestamp: "2026-07-28T15:45:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:24Z" name: isvc-sklearn-graph-2-predictor.18c67e8a537fb1a6 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 50fbed0f-3a2d-4ec5-9d73-1152c0f3152a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 3df20b59-b6ba-4a12-ae2c-e53c5212ca99 kind: Event lastTimestamp: "2026-07-28T15:44:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:51Z" name: isvc-sklearn-graph-2.18c67e7fd6f24bce namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: d53a834e-f5b4-4f51-a4cf-87826cb05a4d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 3df20b59-b6ba-4a12-ae2c-e53c5212ca99 kind: Event lastTimestamp: "2026-07-28T15:44:51Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T15:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:51Z" name: isvc-sklearn-graph-2.18c67e976bc4c6c4 namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: adfa904e-025e-47fe-baed-921018a1bc61 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:43:09.765484Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-f7v2d to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:43:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e7fd308bf67 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: e174fe0d-cb69-4e7c-984a-c58773115c7a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:43:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e7fed829a1a namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: be04ed72-d046-4cf7-81a0-54b02e9f6027 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:06e49304d7aff24f41f8f0b7d4ea2dcf886a7d2132c6967087fee7fad06051a4" already present on machine metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e7feee7f968 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: b7733749-69be-4d98-bc79-cbd3a63bae92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e7ff2c6a671 namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: f9697dbc-bd0a-4427-9197-8d171cab2034 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e7ff32c6bf1 namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: b1cdf0d5-c364-49ca-ac88-5823c2fcadae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:15Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T15:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e8111963bb7 namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 3cd888ee-10fc-43b2-95e7-9fe2f346e22f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.792s (21.792s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e862483d8a7 namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: 40b82737-a4d5-4979-b07b-ef91b9f98f46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e862aed5a2c namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 70d2d653-3222-45a8-936a-1a292cd68730 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e862cf5ca5e namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: f5606d8b-5d44-427c-95da-5f6d52c872a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:37Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T15:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e862d06eadd namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: c80d04d2-ca10-457a-bea5-72aec0453d0b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.949s (2.949s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e86dcd43e1c namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 2d6d13af-9bfb-4c0b-bb86-9f5fad779719 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e86e1b9b9f2 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: a17c8388-0673-477e-ae62-a0759e9c4a39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e86e3481d7e namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: d620c91e-020f-4f92-953e-66d0b67b8c05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:44:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67e86ee398623 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: a746ec25-39ba-42a5-b83a-512841849706 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67ea05f24b9c6 namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 89490e0a-e0ad-411b-ba1b-0f129fe66700 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67ea05f252e16 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 1b5a5ea9-0ef9-4e57-b6fd-103c59b9813d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: abc85371-b616-460b-a02b-47e159b61974 kind: Event lastTimestamp: "2026-07-28T15:45:32Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-f7v2d.18c67ea1028f1dc3 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 2554d3f4-0709-4aaf-a531-721839e9a3b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: 783a994f-aff8-49cd-97a7-a56172b81310 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-f7v2d' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c67e7fd28db336 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 5e5f4537-66d1-40eb-bee0-787b4846e8fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 02f99635-96a2-4e26-a66a-e8ea3c749b72 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: isvc-xgboost-graph-predictor.18c67e7fcda03961 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: c7c19222-6b37-42c3-a564-61476bf54ea8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 533fc304-49c0-426d-95ee-ce11d48fd97d kind: Event lastTimestamp: "2026-07-28T15:44:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:09Z" name: isvc-xgboost-graph-predictor.18c67e834b945152 namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: e62088e3-dcf5-416a-bf15-288f4fa13dc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 533fc304-49c0-426d-95ee-ce11d48fd97d kind: Event lastTimestamp: "2026-07-28T15:44:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:09Z" name: isvc-xgboost-graph-predictor.18c67e834bfed8e2 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: c66675d0-4907-4346-8d23-230994fc6052 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 533fc304-49c0-426d-95ee-ce11d48fd97d kind: Event lastTimestamp: "2026-07-28T15:45:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:09Z" name: isvc-xgboost-graph-predictor.18c67e9146acd29f namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: af5f7e3b-c21b-445b-a739-54bff8c906f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 533fc304-49c0-426d-95ee-ce11d48fd97d kind: Event lastTimestamp: "2026-07-28T15:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:09Z" name: isvc-xgboost-graph-predictor.18c67e914710dcb0 namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: d2cb3b23-7538-40bb-9dcb-e327e9babf84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: b942bf11-0027-4988-bd06-9c30e37b2995 kind: Event lastTimestamp: "2026-07-28T15:44:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:37Z" name: isvc-xgboost-graph.18c67e7fcd4d8ad0 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 02ce828c-8c56-4a51-b6f6-5edb782fa55a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: b942bf11-0027-4988-bd06-9c30e37b2995 kind: Event lastTimestamp: "2026-07-28T15:44:37Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T15:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:37Z" name: isvc-xgboost-graph.18c67e943afeac0d namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 05f7ad0d-7bca-470e-870d-45d3fbe39a6b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:45:19.228013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5578dcbdf9-55mvt to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer-5578dcbdf9-55mvt.18c67e9df79a226d namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: ef557a49-f669-40a8-9997-d34b193e33af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer-5578dcbdf9-55mvt.18c67e9e08441190 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: d6324c81-ed3b-4747-a493-24948c2bc7ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:20Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:45:20Z" name: model-chainer-5578dcbdf9-55mvt.18c67e9e34cbdcfa namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: a271055b-7170-4752-9344-30f21bd1b52c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:20Z" name: model-chainer-5578dcbdf9-55mvt.18c67e9e361958f5 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: ab82fe35-753e-4674-adfb-cb126c31950b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:20Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T15:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:20Z" name: model-chainer-5578dcbdf9-55mvt.18c67e9e39aadfd5 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 1bf9bdb0-2002-4e0a-8856-4d0ab160793d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:20Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T15:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:20Z" name: model-chainer-5578dcbdf9-55mvt.18c67e9e3a27b9ad namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 4cf5fccb-55ab-4388-8471-4475973b96ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: model-chainer-5578dcbdf9-55mvt.18c67ea050f95924 namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: 4154d320-49a1-44cf-af46-3fa247991550 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5578dcbdf9-55mvt namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: bf6990a5-7a7f-4dc8-8b2f-f88d42575a47 kind: Event lastTimestamp: "2026-07-28T15:45:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T15:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:56Z" name: model-chainer-5578dcbdf9-55mvt.18c67ea0cd7e298a namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: d4d3f9a5-e953-42cc-b927-228f98755509 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5578dcbdf9 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 8ee3f61f-1450-4554-b15c-1c48dec07267 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'Created pod: model-chainer-5578dcbdf9-55mvt' metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer-5578dcbdf9.18c67e9df72abc25 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 4adb6628-28aa-446c-a327-e94f6982d4f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: e4e8756b-5977-4dc9-960b-1d48091d6258 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: Scaled up replica set model-chainer-5578dcbdf9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer.18c67e9df65a7f3a namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: bd91cde4-5da2-4e7b-88fb-d7a1ab4c759d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: d8b9fd9d-9f66-4887-b608-9cc176941127 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer.18c67e9dfad9157f namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: c0bd18f6-d9ec-4735-baeb-5a81f001a845 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: d8b9fd9d-9f66-4887-b608-9cc176941127 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer.18c67e9dfe10cd32 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 56b7fc88-f92e-4cdb-b143-cf8f412397f4 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: d8b9fd9d-9f66-4887-b608-9cc176941127 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:19Z" name: model-chainer.18c67e9dfe114141 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 2bf204ca-c7d9-4a67-8f70-1e93806f8cd8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:55:44.596846Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-12202-954b76fdc-jkkll to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:55:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:55:44Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f2f927d4903 namespace: kserve-ci-e2e-test resourceVersion: "21285" uid: 4f6516d4-f4c3-4ef8-aa71-e433f4db6c5a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: "2026-07-28T15:55:45Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:55:45Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f2fac1f627d namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 0e43bf5c-d009-40d0-b61f-0f4553ff598c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12202} kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: "2026-07-28T15:55:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:45Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f2fadbe0e8f namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: d49d0e7a-a2a9-4975-bf7d-3e3594613877 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12202} kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: "2026-07-28T15:55:45Z" message: 'Created container: sequence-graph-12202' metadata: creationTimestamp: "2026-07-28T15:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:45Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f2fb0aca34f namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 1897060c-452d-4072-9243-7b3aa361939e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12202} kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: "2026-07-28T15:55:45Z" message: Started container sequence-graph-12202 metadata: creationTimestamp: "2026-07-28T15:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:45Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f2fb121e01f namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: be6aa485-0a66-4f79-8c98-8e6a01995b00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12202} kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: Stopping container sequence-graph-12202 metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:54Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f31e98cd23c namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 92657930-6551-423b-8d12-56dd530c57fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-12202} kind: Pod name: sequence-graph-12202-954b76fdc-jkkll namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 5fbdb8d0-f4cc-42d9-988c-b741fa57502b kind: Event lastTimestamp: "2026-07-28T15:56:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:21Z" name: sequence-graph-12202-954b76fdc-jkkll.18c67f32656c35f8 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 836bde56-c0e8-4639-b596-2645fdd89948 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-12202-954b76fdc namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 9a0313b6-8395-4bd7-aa7d-a9162ea254c1 kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: 'Created pod: sequence-graph-12202-954b76fdc-jkkll' metadata: creationTimestamp: "2026-07-28T15:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:44Z" name: sequence-graph-12202-954b76fdc.18c67f2f91b8a006 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: a7e86f2a-9798-425f-ac52-b37cb830c2d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-12202 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 8ae4a3ec-4516-4df6-b210-4bfb73e804b5 kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: Scaled up replica set sequence-graph-12202-954b76fdc from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:44Z" name: sequence-graph-12202.18c67f2f9028b063 namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 94173ca4-2352-470d-af7e-9cf7684b36b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-12202 namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: 0494655a-1c5a-428a-bf5b-cb7bdbc45041 kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: InferenceGraph [sequence-graph-12202] is Ready metadata: creationTimestamp: "2026-07-28T15:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:44Z" name: sequence-graph-12202.18c67f2f999842d4 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: e1a86b9b-f6a6-4d6e-9c60-b5a6dc2ee6dc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:56:54.947485Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-50d42-799cd94864-7vggx to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:56:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:56:54Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67f3ff3b6d049 namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: 9984e9c7-87dc-48b5-b9b5-8be095a1f3df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T15:56:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-50d42-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:55Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67f3fffca2123 namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 32122eb9-9429-4d8c-8e53-33909b90012f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T15:56:55Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:56:55Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67f403162d90a namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: cb64f285-256e-4d77-a08e-c9bb4b29fd27 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-50d42} kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T15:56:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:56Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67f40329d5356 namespace: kserve-ci-e2e-test resourceVersion: "22129" uid: 3db854a3-7fb2-4230-94e0-9ff25a7f0104 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-50d42} kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T15:56:56Z" message: 'Created container: sequence-graph-50d42' metadata: creationTimestamp: "2026-07-28T15:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:56Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67f40358820ce namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: 34b54612-5481-4795-a65c-c146b58b49ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-50d42} kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T15:56:56Z" message: Started container sequence-graph-50d42 metadata: creationTimestamp: "2026-07-28T15:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:56:56Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67f4035fbb871 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 04b8e80f-4869-4d19-b5a9-37b15eb8b905 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-50d42} kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Stopping container sequence-graph-50d42 metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:09Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67fb31f8c2584 namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: 8dbed9f3-49b1-459b-947d-934e0220d767 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-50d42} kind: Pod name: sequence-graph-50d42-799cd94864-7vggx namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 3dafbfb3-c9a9-4c29-93c3-aac3353e2452 kind: Event lastTimestamp: "2026-07-28T16:05:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:38Z" name: sequence-graph-50d42-799cd94864-7vggx.18c67fb3ea97ec3f namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: 2805b09d-90f2-4b6c-a06d-7284bb32862f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-50d42-799cd94864 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 56acf8ed-2996-4bd2-ab7f-fb5d542e62a2 kind: Event lastTimestamp: "2026-07-28T15:56:54Z" message: 'Created pod: sequence-graph-50d42-799cd94864-7vggx' metadata: creationTimestamp: "2026-07-28T15:56:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:54Z" name: sequence-graph-50d42-799cd94864.18c67f3ff342bb3d namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: 945dac8f-e270-4a05-9b92-001e3934e9fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-50d42 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 56d441d5-4281-4f05-86f2-06800e9f0999 kind: Event lastTimestamp: "2026-07-28T15:56:54Z" message: Scaled up replica set sequence-graph-50d42-799cd94864 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:56:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:54Z" name: sequence-graph-50d42.18c67f3ff295aa7a namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: cc0f5fe5-25bd-4692-91a9-837767900d7f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-50d42 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: 2d0c606e-e081-4a0c-9cd1-7edaad64d892 kind: Event lastTimestamp: "2026-07-28T15:56:55Z" message: InferenceGraph [sequence-graph-50d42] is Ready metadata: creationTimestamp: "2026-07-28T15:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:55Z" name: sequence-graph-50d42.18c67f3ff73cc029 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 21326dbc-1b94-4a40-bfcb-3f245775589b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:57:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-50d42 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: d1f7d6dd-81f0-4596-a75c-a18badf6ad13 kind: Event lastTimestamp: "2026-07-28T15:57:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:57:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:57:24Z" name: sequence-graph-50d42.18c67f43716238a1 namespace: kserve-ci-e2e-test resourceVersion: "22341" uid: c80f86e6-6f20-4264-9769-6b3fa00c8524 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:57:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-50d42 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: d1f7d6dd-81f0-4596-a75c-a18badf6ad13 kind: Event lastTimestamp: "2026-07-28T15:57:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:57:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:57:24Z" name: sequence-graph-50d42.18c67f4371ea08c7 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 0340210d-a2c9-44e8-8b8c-3b22b061fbf5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:46:29.546246Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-642bb-65f5dc7d74-5xcjh to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:46:29Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67eae56e5392b namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: 0482103b-7c0a-47ed-bad5-11b0891ac9da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:46:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-642bb-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:46:29Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67eae618d2fff namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 94af016e-7885-46ca-a034-6dbda264c696 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16840" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:46:30Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:46:30Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67eae93eac5d3 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: c63338f7-291f-46c6-bbf9-4a5379efa2c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-642bb} kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:46:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:46:30Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67eae9540805a namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 3c478d9a-278b-49e4-a31f-a18dda92f176 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-642bb} kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:46:30Z" message: 'Created container: sequence-graph-642bb' metadata: creationTimestamp: "2026-07-28T15:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:46:30Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67eae97e21078 namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: 7da61b84-7b97-4260-a781-e10ea8ee9b23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-642bb} kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:46:30Z" message: Started container sequence-graph-642bb metadata: creationTimestamp: "2026-07-28T15:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:46:30Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67eae984fa452 namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: 37df43ae-6567-4a3b-987e-8dd8d2fd5971 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-642bb} kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: Stopping container sequence-graph-642bb metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:44Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67f218ac7c9e1 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: e15adbc3-ebb3-473f-afc3-ecf7a67f6029 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-642bb} kind: Pod name: sequence-graph-642bb-65f5dc7d74-5xcjh namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 180c1216-cc9b-4404-afc1-9d206f5bbeb2 kind: Event lastTimestamp: "2026-07-28T15:55:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T15:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:11Z" name: sequence-graph-642bb-65f5dc7d74-5xcjh.18c67f22166dbe35 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 168940e1-7d48-429c-81d7-a5d380129d85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-642bb-65f5dc7d74 namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: 53183a90-d674-416a-84cf-8249e374224e kind: Event lastTimestamp: "2026-07-28T15:46:29Z" message: 'Created pod: sequence-graph-642bb-65f5dc7d74-5xcjh' metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:29Z" name: sequence-graph-642bb-65f5dc7d74.18c67eae56567d84 namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: e04da752-f9aa-42f8-90b7-03e626ac8af7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-642bb namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: e02615fe-528b-4585-b4ef-3a430a896d14 kind: Event lastTimestamp: "2026-07-28T15:46:29Z" message: Scaled up replica set sequence-graph-642bb-65f5dc7d74 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:29Z" name: sequence-graph-642bb.18c67eae55940f9c namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 0ec5981a-4aea-4498-b843-eb295b2cbbbd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-642bb namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: eb00b61b-7c78-49d9-bea8-16ad3cf043ac kind: Event lastTimestamp: "2026-07-28T15:46:29Z" message: InferenceGraph [sequence-graph-642bb] is Ready metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:46:29Z" name: sequence-graph-642bb.18c67eae5a5fe25a namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: f29905e3-bba1-4bf0-995c-0e5e5ddfc082 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:46:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-642bb namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: f1f995d1-c03e-48e4-8f01-a81ac764f584 kind: Event lastTimestamp: "2026-07-28T15:46:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:59Z" name: sequence-graph-642bb.18c67eb1d471d6dc namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: 39ab0fbe-b9d6-47e9-810f-fcd46c7f7794 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:46:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-642bb namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: f1f995d1-c03e-48e4-8f01-a81ac764f584 kind: Event lastTimestamp: "2026-07-28T15:46:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:59Z" name: sequence-graph-642bb.18c67eb1d4d0db8a namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: b4d0d743-cb27-4080-981f-f4fca1f9ae3d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:05:39.788364Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26390" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b87a7-758cb69b74-ndv9b to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:05:39Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fba26ab11bc namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: 43260d8f-252a-422d-905f-3303570b02ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-b87a7-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:39Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fba32a6b6f9 namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 60a4209a-6fcd-4a73-ad60-483d4bca056c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:40Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26403" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:05:40Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:05:40Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fba64522878 namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 62b32819-1be5-451c-b5dd-210c434eb5b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b87a7} kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:05:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:40Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fba66014400 namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: c227e12d-645b-4a71-94aa-a72e1aca852b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b87a7} kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:05:40Z" message: 'Created container: splitter-graph-b87a7' metadata: creationTimestamp: "2026-07-28T16:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:40Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fba6949c814 namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: 12ac9360-c726-4b6a-ada4-738f5e5ca432 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b87a7} kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:05:40Z" message: Started container splitter-graph-b87a7 metadata: creationTimestamp: "2026-07-28T16:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:40Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fba69bd35ed namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 3c361be1-b8bb-4f54-8619-e0ff1bf8ca8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b87a7} kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:05:49Z" message: Stopping container splitter-graph-b87a7 metadata: creationTimestamp: "2026-07-28T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:49Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fbc7e8cb2be namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 4d8641c3-7a99-459f-9982-b144b274e12a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b87a7} kind: Pod name: splitter-graph-b87a7-758cb69b74-ndv9b namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 2b776be4-14a7-40af-82f1-9c44880461bf kind: Event lastTimestamp: "2026-07-28T16:06:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:16Z" name: splitter-graph-b87a7-758cb69b74-ndv9b.18c67fbcfbe7a7a3 namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 1f299724-c4f4-4b18-9688-d3fe9df5e37f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b87a7-758cb69b74 namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: b093035e-3cb9-4dee-a42c-42520d71047c kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'Created pod: splitter-graph-b87a7-758cb69b74-ndv9b' metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: splitter-graph-b87a7-758cb69b74.18c67fba25b1ef99 namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: 7ddbe780-45a0-46ac-bcbf-2cae202123c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: fea0320d-b2fe-465f-aaae-9d207afba84f kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: Scaled up replica set splitter-graph-b87a7-758cb69b74 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: splitter-graph-b87a7.18c67fba24bd1be6 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 400319b7-8625-4ef5-bb7b-e886481b49c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 727a4b34-9030-40e1-9117-33f301eb2933 kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: InferenceGraph [splitter-graph-b87a7] is Ready metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:39Z" name: splitter-graph-b87a7.18c67fba2c984c30 namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 64119b00-fd80-456e-91cd-1e094ded9c54 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:06:50.119747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27199" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-dd6ec-8474d74dd-rx4fl to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c67fca86bed80e namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: 0b08f077-4856-42b7-8a56-da7a885c5a36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c67fcaa05eb90c namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: 7b88a3c2-fdb9-4e68-95ff-e1a54739fff5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-dd6ec} kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c67fcaa1c6b361 namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: 6ad07099-10e6-4189-ae0c-c177a5828be6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-dd6ec} kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: 'Created container: splitter-graph-dd6ec' metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c67fcaa55c39b9 namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: 761f8cd0-fe16-4edf-93ce-6b5f24a91351 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-dd6ec} kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: Started container splitter-graph-dd6ec metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c67fcaa5d1afc9 namespace: kserve-ci-e2e-test resourceVersion: "27226" uid: 7d1af156-65f0-4ec7-a668-0c354a23b2e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-dd6ec} kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: "2026-07-28T16:15:04Z" message: Stopping container splitter-graph-dd6ec metadata: creationTimestamp: "2026-07-28T16:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:04Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c6803dac97413a namespace: kserve-ci-e2e-test resourceVersion: "30640" uid: 5faafdb4-c0ba-4d2b-ba4e-872530534a0b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:15:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-dd6ec} kind: Pod name: splitter-graph-dd6ec-8474d74dd-rx4fl namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: ddb114e8-aac1-4ca8-abec-4af73f0b8b2e kind: Event lastTimestamp: "2026-07-28T16:15:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:15:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:32Z" name: splitter-graph-dd6ec-8474d74dd-rx4fl.18c6803e44d8ea5c namespace: kserve-ci-e2e-test resourceVersion: "30875" uid: 83f2dda1-71d8-4664-9ade-2cfbe1c71842 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-dd6ec-8474d74dd namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: 5bfcfea2-a03f-4852-b44a-361557dd9cde kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: 'Created pod: splitter-graph-dd6ec-8474d74dd-rx4fl' metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec-8474d74dd.18c67fca86559491 namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: 6da123f8-fd31-424d-91c7-17a9b6ec64c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "27192" uid: d2f9392b-6c5b-47c0-a0b8-cb49baf53648 kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: Scaled up replica set splitter-graph-dd6ec-8474d74dd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec.18c67fca856dfb3c namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: d2204bb9-3d2e-4a10-a27c-248a3f135ebd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "27212" uid: ace358d0-7927-401b-bd04-892e8af30514 kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: InferenceGraph [splitter-graph-dd6ec] is Ready metadata: creationTimestamp: "2026-07-28T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:06:50Z" name: splitter-graph-dd6ec.18c67fca897f397d namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: 14f00da0-6c36-46f5-bf66-8b6c8e0e9de4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:07:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "27194" uid: 948ec8f3-f62c-4df8-a649-7cad90568395 kind: Event lastTimestamp: "2026-07-28T16:07:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:07:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:07:20Z" name: splitter-graph-dd6ec.18c67fce03e897a4 namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: 86679f53-6bbc-461a-b8b9-b7aac464efd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:07:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "27194" uid: 948ec8f3-f62c-4df8-a649-7cad90568395 kind: Event lastTimestamp: "2026-07-28T16:07:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:07:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:07:20Z" name: splitter-graph-dd6ec.18c67fce043dfc36 namespace: kserve-ci-e2e-test resourceVersion: "27444" uid: ebc7a7b5-5dad-4a25-9fc1-1c0242300076 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:54:44.604647Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-12202-predictor-f74599657-2zkrq to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:54:44Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f219aad062a namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: a42dcfbf-ac7b-42bc-b2a9-0908566777ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-12202-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:44Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21a5647d13 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 8837620f-04f9-41b0-abef-19239d4b2d2e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21d813ea1a namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: cf72f7c8-1d45-481c-8257-2a140ea201e2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21d9d5b8e9 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 51dd8aad-bcf3-445e-824c-f01ea29b71f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21dce1cdef namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 6bb45934-e85b-486c-bdd3-2938df7777ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21dd50763b namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: c2c36e74-9b19-4ab5-a365-00819e357be7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21dd5a97e1 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 3c4f2ffa-b491-4615-8d3c-3ed55721358b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21e04f5f5e namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 79c69371-7bcd-404a-a8bd-02012669933d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:54:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:45Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f21e14a9ec1 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 494c1600-f7c3-4dea-9d38-e978f452d445 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:55:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:22Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f220724021f namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: f3ab96a2-8b2b-44ed-abe8-c9ea515b6173 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f31f44a2e14 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 43a6427e-35b0-4ddb-a023-b55acaaaecaa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f31f44ad4cf namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 52465a9b-a737-4ef9-afc8-37a9d09559c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12202-predictor-f74599657-2zkrq namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 7c7c7959-59bd-492e-87f4-1099cf7b46b8 kind: Event lastTimestamp: "2026-07-28T15:55:57Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:57Z" name: success-200-isvc-12202-predictor-f74599657-2zkrq.18c67f328f50018b namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: fa3e75eb-d197-40ed-9daa-24e351896bfc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-12202-predictor-f74599657 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: e0c18361-59f0-41df-ba15-29f74f3351d7 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: 'Created pod: success-200-isvc-12202-predictor-f74599657-2zkrq' metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:44Z" name: success-200-isvc-12202-predictor-f74599657.18c67f219a527588 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: f0bfbbea-48d9-4bd7-8999-6b57321db62c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: ae0dff4c-1350-4b8c-8304-bcbe9c334af2 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: Scaled up replica set success-200-isvc-12202-predictor-f74599657 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:44Z" name: success-200-isvc-12202-predictor.18c67f21966b74f1 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 968d3500-07da-46d3-8eb4-74f130aa66fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 231649c6-2626-4295-a3bb-65a0f007cdfd kind: Event lastTimestamp: "2026-07-28T15:55:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:29Z" name: success-200-isvc-12202-predictor.18c67f25158b72c4 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: ace466ff-b081-4d05-9092-3618a5482f6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 231649c6-2626-4295-a3bb-65a0f007cdfd kind: Event lastTimestamp: "2026-07-28T15:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:29Z" name: success-200-isvc-12202-predictor.18c67f25161aa0a4 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 42da239e-5de4-4343-81c1-8b6bace5466d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 231649c6-2626-4295-a3bb-65a0f007cdfd kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:44Z" name: success-200-isvc-12202-predictor.18c67f28962b109a namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 56c816e7-4e17-4fba-b6f4-0ee1cbe4e7f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12202-predictor namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 231649c6-2626-4295-a3bb-65a0f007cdfd kind: Event lastTimestamp: "2026-07-28T15:55:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:44Z" name: success-200-isvc-12202-predictor.18c67f2896c5c725 namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 72774880-a41f-4e3a-b1c7-181f962cc0b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12202 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: c1e56b45-8473-4e2a-93fd-a9feae297917 kind: Event lastTimestamp: "2026-07-28T15:55:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:32Z" name: success-200-isvc-12202.18c67f21968248ba namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: abb39451-e409-4214-9e03-e8c85d48fa5e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12202 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: c1e56b45-8473-4e2a-93fd-a9feae297917 kind: Event lastTimestamp: "2026-07-28T15:55:32Z" message: InferenceService [success-200-isvc-12202] is Ready metadata: creationTimestamp: "2026-07-28T15:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:32Z" name: success-200-isvc-12202.18c67f2cc4e8b653 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 01a7a6fa-e66b-475c-bae2-e5f4c12e519e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:44:49.526440Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4b277-predictor-6cf4674d7-4h57h to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e970d402180 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 42bc579a-c3f4-48b8-ba12-4b5e2930b514 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e972750d246 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 24ffd790-2354-4e0f-8428-ac6c847134c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e9728c2e9f6 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: cd22dbdf-1d75-40be-aebc-1dd01ab6887b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e972ba0cfc5 namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: e3968547-467e-45ee-b360-cf95533c6d03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e972c05f2fe namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 7b0ef172-350b-457c-ae66-07731b3fae55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e972c10a867 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: bf020b65-8326-41a3-b650-1fbb9f6467b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e972efbcfe1 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 4ec958ff-ca71-4a45-9832-e9c72952e78e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:44:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:44:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:50Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e972f628859 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 3b542afb-d19a-4342-a5a0-3ae4e4b6a560 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:44:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:45:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:27Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67e978b15d034 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: cba63b14-566b-42e1-a065-71b1f35e5564 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67f1a9441e9c2 namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: 7fda8daf-777f-410d-a270-1487a98e3789 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67f1a9442695d namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: bda3aa34-c3d2-4cc1-b8b7-bf6bd30a2f03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:54:18Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-28T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:18Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67f1b8f6f3206 namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 7c0f78af-ace7-4a55-bd85-2c466bb76457 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: d26cc127-e2c3-47ac-b9f8-dfcaf4867550 kind: Event lastTimestamp: "2026-07-28T15:54:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: i/o timeout' metadata: creationTimestamp: "2026-07-28T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:18Z" name: success-200-isvc-4b277-predictor-6cf4674d7-4h57h.18c67f1b8fbd7089 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: 2efcdaea-00f7-4a76-b3f8-4737f9038ad1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4b277-predictor-6cf4674d7 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: fd9ac3b2-287d-4719-a863-98c072b26dfd kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: 'Created pod: success-200-isvc-4b277-predictor-6cf4674d7-4h57h' metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277-predictor-6cf4674d7.18c67e970cb78d7a namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 15893848-ccc4-4e94-80aa-087942441dcd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 76dfd4ac-e67d-4733-b5d8-fcb46fc785fc kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Scaled up replica set success-200-isvc-4b277-predictor-6cf4674d7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277-predictor.18c67e970948a985 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: cd8d9db9-76b0-468d-b854-4a2407e90bd5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 85efc237-cbd2-4225-91e5-2fc4e18a1082 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:19Z" name: success-200-isvc-4b277-predictor.18c67e9a88031462 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 0d514043-a5d0-48ee-ae68-8e482ba16623 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:45:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 85efc237-cbd2-4225-91e5-2fc4e18a1082 kind: Event lastTimestamp: "2026-07-28T15:45:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:19Z" name: success-200-isvc-4b277-predictor.18c67e9a88673881 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: 265d6e17-2d5d-4b87-bdf8-6180a78ea89d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:45:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 85efc237-cbd2-4225-91e5-2fc4e18a1082 kind: Event lastTimestamp: "2026-07-28T15:46:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:04Z" name: success-200-isvc-4b277-predictor.18c67ea1863f8c09 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 254b9e75-640d-4e6e-9e87-54038ba29c79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:45:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b277-predictor namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 85efc237-cbd2-4225-91e5-2fc4e18a1082 kind: Event lastTimestamp: "2026-07-28T15:46:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:04Z" name: success-200-isvc-4b277-predictor.18c67ea186cf762e namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 8f3a06a1-f78a-4b17-b055-51f7a7dfbb22 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 8f033c47-bc06-4134-9f5a-0ca9dcfe62c5 kind: Event lastTimestamp: "2026-07-28T15:45:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:37Z" name: success-200-isvc-4b277.18c67e9709c8639d namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 4376d63e-d3dc-4803-9105-f7a0496a9847 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 8f033c47-bc06-4134-9f5a-0ca9dcfe62c5 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-4b277": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4b277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277.18c67e970a627cd2 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 23cf2705-1f55-470e-811e-ad05ff71f651 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 8f033c47-bc06-4134-9f5a-0ca9dcfe62c5 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4b277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-4b277.18c67e970a62d241 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 3b6ae507-8762-4ce9-bd80-47dfe88ce300 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b277 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 8f033c47-bc06-4134-9f5a-0ca9dcfe62c5 kind: Event lastTimestamp: "2026-07-28T15:45:37Z" message: InferenceService [success-200-isvc-4b277] is Ready metadata: creationTimestamp: "2026-07-28T15:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:37Z" name: success-200-isvc-4b277.18c67ea247db3add namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 44de97d2-0cd0-4b4c-beb7-1ed80a100c63 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:55:54.936113Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-50d42-predictor-554687cbf-4nw4f to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f31fac1f48c namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 52269d04-59a2-49c7-9a02-3894b2fb19a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-50d42-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:55Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f320417b3fb namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 29b00d4b-f1f1-4bf4-be15-31b1474ffbe9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:55:55Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f323833dadd namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: 4bc4904a-f61a-43b0-b9bf-695f7b53b20d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:56Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f323a2a292e namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 3996b876-e6a0-4daa-ae3d-c2fda71dba08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:56Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f323d2280e3 namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: a9eaf6af-980d-42e0-84f5-c2933803b13c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:56Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f323d8fbd6e namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 676485cd-b7a5-45d5-b360-173d834e02ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:56Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f323d993174 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 6b43e5dc-b8bc-4fb9-867e-515890f22c8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:56Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f32413c5fe7 namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 504251fa-68f0-4691-8d21-ef2bf8e4d7fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T15:55:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:56Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f3241ab7579 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: e60f4677-d690-4669-9546-da49d59e1da3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T16:05:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:12Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67f32678b36b4 namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: 90082f6a-3478-42e7-a864-78c80797158f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67fb3272d5799 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 3d1df50c-f3ff-44ac-9865-d61064111a3c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67fb3272db28d namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: a6ad2f0e-d029-4f6a-b89e-b522601400a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50d42-predictor-554687cbf-4nw4f namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: eaca4477-e19f-4dd6-9048-5ea6a3f63b40 kind: Event lastTimestamp: "2026-07-28T16:05:12Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:12Z" name: success-200-isvc-50d42-predictor-554687cbf-4nw4f.18c67fb3dbef22d5 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 765520ff-7554-41c8-81c2-abb90af4f404 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-50d42-predictor-554687cbf namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 07747e51-c05e-4913-8223-38f3e29436c3 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: 'Created pod: success-200-isvc-50d42-predictor-554687cbf-4nw4f' metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-50d42-predictor-554687cbf.18c67f31f9de983e namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: dd00dfa1-0a16-465a-a6ef-da12bbd7391b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: ce5361b3-9666-400d-8f2a-7a4fe21dd758 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: Scaled up replica set success-200-isvc-50d42-predictor-554687cbf from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-50d42-predictor.18c67f31f5b6aa0c namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 0590e4c8-8de2-48ad-aed4-dd1d5e5d8622 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:56:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: acb262cd-0d3c-4262-ba3d-e5132f71ef83 kind: Event lastTimestamp: "2026-07-28T15:56:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:24Z" name: success-200-isvc-50d42-predictor.18c67f35785a2c77 namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: f4aa5605-d0f1-41a0-9769-b20f76c11074 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:56:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: acb262cd-0d3c-4262-ba3d-e5132f71ef83 kind: Event lastTimestamp: "2026-07-28T15:56:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:24Z" name: success-200-isvc-50d42-predictor.18c67f35795cb376 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: e9a53743-2274-481b-a68f-daa027438a83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: acb262cd-0d3c-4262-ba3d-e5132f71ef83 kind: Event lastTimestamp: "2026-07-28T15:57:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:57:25Z" name: success-200-isvc-50d42-predictor.18c67f3c78138e8f namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: 15a17e07-fa26-4113-84d7-546c02b35974 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50d42-predictor namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: acb262cd-0d3c-4262-ba3d-e5132f71ef83 kind: Event lastTimestamp: "2026-07-28T15:57:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:57:25Z" name: success-200-isvc-50d42-predictor.18c67f3c787e4232 namespace: kserve-ci-e2e-test resourceVersion: "22345" uid: 344caa17-e06e-469a-8efe-f2fca8a83e34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:55:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-50d42 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: d0fab7c9-aa44-4dcb-aed9-b63b30e2b888 kind: Event lastTimestamp: "2026-07-28T15:56:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:42Z" name: success-200-isvc-50d42.18c67f31f5ebc937 namespace: kserve-ci-e2e-test resourceVersion: "21987" uid: 68b3840d-bc91-487f-8455-97a3459ef586 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-50d42 namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: d0fab7c9-aa44-4dcb-aed9-b63b30e2b888 kind: Event lastTimestamp: "2026-07-28T15:56:42Z" message: InferenceService [success-200-isvc-50d42] is Ready metadata: creationTimestamp: "2026-07-28T15:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:42Z" name: success-200-isvc-50d42.18c67f3d2428c483 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: b68826c3-ede2-4d01-bfab-5306ee7b3f4b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:45:29.518278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-642bb-predictor-b88b6c887-hlk59 to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:45:29Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea05cf31aa6 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: ff7d95a6-7c07-4a7e-8384-950deaa1a424 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-642bb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:29Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea067b62e25 namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 6221134f-2cd7-4db0-bf6d-82972921ad99 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea09af175f5 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: a26c251f-5769-4923-9969-58ec967ccb9f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea09c48b202 namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: d782ab71-765d-4c8a-b5c9-2b5c5c919ec5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea09face082 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: c333106c-5f82-4827-902e-9473808f7c4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea0a0354c83 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 09eb9eb8-0811-4c7f-a0be-b4a983f27980 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea0a03efe39 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 183e16a9-019f-4223-acb6-8098c5cee489 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea0a336bcc5 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 774caa8c-0b71-4d11-868c-6aef7b12f077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:45:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:30Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea0a39b343d namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: cb0fc1de-469c-4051-abb7-b16d5cc5dbdb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:54:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:47Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67ea0e7913e19 namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: 2ff6312a-2731-4551-90a0-4f16641ea3bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:44Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67f219c247549 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: b45175c3-b2f4-4293-9f69-d3abe2ca858c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:54:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:44Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67f219c24cd89 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: ba55254e-f173-442e-a1d1-edc2ce2256ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-642bb-predictor-b88b6c887-hlk59 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 566fe303-800a-4a98-9a8b-f8fc3e2681e3 kind: Event lastTimestamp: "2026-07-28T15:54:47Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:47Z" name: success-200-isvc-642bb-predictor-b88b6c887-hlk59.18c67f225bfc0ab6 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: a713abd2-bf85-489c-be5b-6157d11a513e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-642bb-predictor-b88b6c887 namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: a8774346-64b5-4b12-99ad-da0d9233f77b kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: 'Created pod: success-200-isvc-642bb-predictor-b88b6c887-hlk59' metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:29Z" name: success-200-isvc-642bb-predictor-b88b6c887.18c67ea05c7d195e namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 093bfe41-15c0-4f95-8ed1-1f86fc8701aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: edcff3c5-c75b-4916-aee4-d16d551cb265 kind: Event lastTimestamp: "2026-07-28T15:45:29Z" message: Scaled up replica set success-200-isvc-642bb-predictor-b88b6c887 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:29Z" name: success-200-isvc-642bb-predictor.18c67ea05a9d3da9 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 39761904-f43f-448d-88a3-025c8762549f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:45:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: ac5a6ea7-a6d4-4067-8c6b-b4a6b7023ef7 kind: Event lastTimestamp: "2026-07-28T15:46:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:14Z" name: success-200-isvc-642bb-predictor.18c67ea3d98578d1 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: c4cae153-b17f-4e95-a53b-b5d250bf4c2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:45:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: ac5a6ea7-a6d4-4067-8c6b-b4a6b7023ef7 kind: Event lastTimestamp: "2026-07-28T15:46:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:14Z" name: success-200-isvc-642bb-predictor.18c67ea3d9e81b48 namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: 9aeb00ef-bb5b-4cbc-8ccd-10be065f5e83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:46:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: ac5a6ea7-a6d4-4067-8c6b-b4a6b7023ef7 kind: Event lastTimestamp: "2026-07-28T15:46:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:59Z" name: success-200-isvc-642bb-predictor.18c67eae56876291 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: 89a007e0-c887-4d2b-8e8c-764aefb9fb24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T15:46:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-642bb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: ac5a6ea7-a6d4-4067-8c6b-b4a6b7023ef7 kind: Event lastTimestamp: "2026-07-28T15:46:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:59Z" name: success-200-isvc-642bb-predictor.18c67eae573c5f33 namespace: kserve-ci-e2e-test resourceVersion: "17085" uid: 60b17e3d-641f-4f56-9a14-9b6a339b14c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:45:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-642bb namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: aaa5182f-7516-4e65-ac31-1b5410a5e220 kind: Event lastTimestamp: "2026-07-28T15:46:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:46:17Z" name: success-200-isvc-642bb.18c67ea05ab6973c namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 910c18fe-6772-42bf-9601-8ee9879121ef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:46:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-642bb namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: aaa5182f-7516-4e65-ac31-1b5410a5e220 kind: Event lastTimestamp: "2026-07-28T15:46:17Z" message: InferenceService [success-200-isvc-642bb] is Ready metadata: creationTimestamp: "2026-07-28T15:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:46:17Z" name: success-200-isvc-642bb.18c67eaba35ab6b5 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 50315360-ffaf-40a8-9ab0-95939a119b7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:43:09.170528Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:43:09Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e7faf92788f namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 762898c0-cdc7-4351-8d26-cdd149fb7335 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8fc0c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:09Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e7fc07b28de namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: a7059db3-3f16-4504-8889-9f7f47486845 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:43:10Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e7fee67e51d namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 3cd1a403-9ecd-428a-8c81-ed789664f868 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:10Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:10Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e7fefd0fc76 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: e33c8f72-b681-4eb2-a8d9-d4b7d4ed2a4b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:28Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" in 17.995s (17.995s including waiting). Image size: 1335001715 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:28Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e8420704d53 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: cd54b957-f2b1-49b7-b1cf-de02ec985438 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e860eff1292 namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 70c178d1-d441-4988-a96b-6fa895d11510 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e861060275d namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 941b52bf-6982-4773-8164-ab0118410b5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T15:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:36Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e86106f8d76 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: af2f86aa-d244-46f6-b315-dc397d59dc54 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.527s (3.527s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e86e2adacbc namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 634baf9b-814b-40f9-b77e-793005d03170 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e86e6db10b4 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: f575c9fb-e443-4d43-816b-3dd082fe9abb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:43:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:43:40Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e86e7f25378 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: fa2e7fd9-ff5b-4e43-82dc-ae2ed9bee84a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:44:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:17Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e86ee3775ba namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 3c9e5cf6-4c62-4c11-b3c4-c70eb31219db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e9709e44447 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: db5323c4-005b-465c-b189-ef6a38f2ad23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:49Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e9709e4dbda namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 073e8588-c133-4d1f-98ca-43c80fcef58a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 16e84937-6958-481b-8304-422535a23446 kind: Event lastTimestamp: "2026-07-28T15:44:52Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:52Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw.18c67e97b25b452d namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 49ea00fe-7d2e-4de7-8e50-a1138895bbf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8fc0c-predictor-5697fc6c87 namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: f1fb5381-e3f0-4e1f-95db-30f886cac1e4 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: 'Created pod: success-200-isvc-8fc0c-predictor-5697fc6c87-29ghw' metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: success-200-isvc-8fc0c-predictor-5697fc6c87.18c67e7faf21872b namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: e5fd3444-f806-4c48-b206-1159888d30ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14467" uid: 2e8a68b5-4992-4857-afdb-fd7cc0fc8582 kind: Event lastTimestamp: "2026-07-28T15:43:09Z" message: Scaled up replica set success-200-isvc-8fc0c-predictor-5697fc6c87 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:43:09Z" name: success-200-isvc-8fc0c-predictor.18c67e7fab6c3147 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: b1c81df4-a86f-48a5-82a2-82f06df87514 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: 16cc0465-8d7d-43d8-bdf1-ccf81d8c458c kind: Event lastTimestamp: "2026-07-28T15:44:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:09Z" name: success-200-isvc-8fc0c-predictor.18c67e8330a10419 namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: 6948da4d-31ff-4a94-b445-6f5ff88ea3ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T15:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: 16cc0465-8d7d-43d8-bdf1-ccf81d8c458c kind: Event lastTimestamp: "2026-07-28T15:44:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:09Z" name: success-200-isvc-8fc0c-predictor.18c67e8331073d7e namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: e1dd7e9c-7d51-418c-9a9c-586141fa4523 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 16cc0465-8d7d-43d8-bdf1-ccf81d8c458c kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:39Z" name: success-200-isvc-8fc0c-predictor.18c67e912ce46246 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: d8650985-3a83-438d-ad72-7a8ee13e537b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:44:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fc0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 16cc0465-8d7d-43d8-bdf1-ccf81d8c458c kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:39Z" name: success-200-isvc-8fc0c-predictor.18c67e912d51a69b namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 3f8bee49-72d3-4fef-95ba-71f3d4e90870 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8fc0c namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 7854d15e-b8d8-45ff-90e2-ab6643d90d46 kind: Event lastTimestamp: "2026-07-28T15:44:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:27Z" name: success-200-isvc-8fc0c.18c67e7fb18ef6e2 namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: d1139f5d-479c-4969-a360-a062bfc79104 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8fc0c namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 7854d15e-b8d8-45ff-90e2-ab6643d90d46 kind: Event lastTimestamp: "2026-07-28T15:44:27Z" message: InferenceService [success-200-isvc-8fc0c] is Ready metadata: creationTimestamp: "2026-07-28T15:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:27Z" name: success-200-isvc-8fc0c.18c67e91e6c78d4e namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: e443ef20-7789-449d-a231-08e2e798e921 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:55:24.963011Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a87c3-predictor-6555844d55-tbqff to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:55:24Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b0038b06c namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: e278072f-bf81-49e5-8745-76d27f682861 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a87c3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b105969a6 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: faff5949-8303-4bc3-9539-d6182f47ecad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b3e086fc2 namespace: kserve-ci-e2e-test resourceVersion: "21075" uid: b4d29b2a-5211-4695-9f46-ad4b16a654ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b3fdd4f69 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 91f142df-a211-4358-b2da-dd3f28ea1b59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b44e014c8 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: ba37d542-c24d-4760-8470-0bb65468f128 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b4555b3ce namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 4f87b342-7f61-4f15-9c59-cd9dc5c4a8ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b4560360a namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 2e16f68e-cb8a-49da-8dd5-ff9222a25e16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b484c9946 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: 2740c05a-ce7b-42f9-9cf1-7e27d5a3a074 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T15:55:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:26Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b48bb289b namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 6402be64-4e58-4195-9ae0-7152c6355d70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T16:04:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:42Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67f2b62b7e0f1 namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: 416b4ed9-771d-41cd-9fdc-0a431a99aa5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67fac264b34b8 namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 71853856-d9af-4066-84a1-bc535a2b28fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67fac264bc52e namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 29583af2-8a8c-46fe-a9a8-79f70a51adb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a87c3-predictor-6555844d55-tbqff namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e176455e-dbd6-4726-afc8-f4fb49e23800 kind: Event lastTimestamp: "2026-07-28T16:04:42Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:42Z" name: success-200-isvc-a87c3-predictor-6555844d55-tbqff.18c67facd7226f8a namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: 6dd86029-4287-40ae-b73d-98868ed5857c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a87c3-predictor-6555844d55 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 055c3636-db39-42d3-9fdf-2be7aadccf15 kind: Event lastTimestamp: "2026-07-28T15:55:24Z" message: 'Created pod: success-200-isvc-a87c3-predictor-6555844d55-tbqff' metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:24Z" name: success-200-isvc-a87c3-predictor-6555844d55.18c67f2afe9bdce2 namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 40d811df-c344-43bf-9372-9b492fa04cf7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 5bf97f58-6b99-4adf-8f97-f63e082b9846 kind: Event lastTimestamp: "2026-07-28T15:55:24Z" message: Scaled up replica set success-200-isvc-a87c3-predictor-6555844d55 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:24Z" name: success-200-isvc-a87c3-predictor.18c67f2afb523111 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: c441c662-2e21-4bdc-9fa3-7339de83409b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 519520d7-1e91-48be-9724-ffe961d07ec7 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-a87c3-predictor.18c67f2e79c96040 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: dc5cbfda-10c2-45ba-9466-ac6b370649c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 519520d7-1e91-48be-9724-ffe961d07ec7 kind: Event lastTimestamp: "2026-07-28T15:55:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:54Z" name: success-200-isvc-a87c3-predictor.18c67f2e7a8c5f2d namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 603a1b88-a5ab-464e-8710-5bcbd0d25ccb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:56:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 519520d7-1e91-48be-9724-ffe961d07ec7 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:25Z" name: success-200-isvc-a87c3-predictor.18c67f357d265f22 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 37b0f5e2-1359-4d7c-be74-7b54ae3218ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:56:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a87c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 519520d7-1e91-48be-9724-ffe961d07ec7 kind: Event lastTimestamp: "2026-07-28T15:56:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:56:25Z" name: success-200-isvc-a87c3-predictor.18c67f357e5ec879 namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: d368dd81-c842-4c36-a615-0826dc25d31a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: ccc09b54-f89b-453d-93f0-09e2e6d2a92b kind: Event lastTimestamp: "2026-07-28T15:56:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:12Z" name: success-200-isvc-a87c3.18c67f2afb4a80c4 namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: ee67b9a6-b387-4fec-83eb-820d92475ba8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:56:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a87c3 namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: ccc09b54-f89b-453d-93f0-09e2e6d2a92b kind: Event lastTimestamp: "2026-07-28T15:56:12Z" message: InferenceService [success-200-isvc-a87c3] is Ready metadata: creationTimestamp: "2026-07-28T15:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:56:12Z" name: success-200-isvc-a87c3.18c67f362018ce9a namespace: kserve-ci-e2e-test resourceVersion: "21691" uid: ac98a8d2-22f5-491e-9766-f5a2e3f70561 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:54:14.508163Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1a98c90fcb namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 03a15349-5de1-4806-8f25-ee78c23a2db4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1ab2904f90 namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: 0595ef3c-6d72-409a-9cd2-4b59c4d97a68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1ab40c7933 namespace: kserve-ci-e2e-test resourceVersion: "20153" uid: d68f7a6c-061e-4621-9977-418df765d0f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1ab70df1e9 namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: 9382d026-fcf2-4684-874f-800e53437bab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1ab7b6b137 namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 15e46651-fe44-4c99-b2c0-6ab406b0b3f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1ab7c0b145 namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: a6540c3f-b5db-4161-b930-6b2901d5be15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1abb80e7fb namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 372b6647-d9c1-419f-be91-025f645fcbe1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:15Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1abbf08fbc namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: d956a67f-240b-4fd2-9141-3e96b229912c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:54:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:52Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f1b024dc625 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 377d8dfc-6357-4271-b5b3-fd3bbdd67113 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f2b05b8bb12 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: a1ff3a3c-01cb-4aee-920c-15165c6449d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:55:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:25Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f2b05b92e43 namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: a3904396-17b6-4cb9-89ba-6b68070f415a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: c7ef5e1a-bb17-48f6-92ff-05e6a48c1163 kind: Event lastTimestamp: "2026-07-28T15:55:27Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T15:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:55:27Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm.18c67f2b8a9b6725 namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 07e7f705-857f-4227-9e0b-989da040f5d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ad35d-predictor-5d7d8b4bcc namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 5f7fc148-8ab3-4e4a-8297-b26947a1af04 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: 'Created pod: success-200-isvc-ad35d-predictor-5d7d8b4bcc-wtmcm' metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d-predictor-5d7d8b4bcc.18c67f1a983a6be0 namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 0663f953-0b78-4330-9490-66552667eae5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 8fcc5dbd-e1be-47d7-b286-0018ed93d23b kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Scaled up replica set success-200-isvc-ad35d-predictor-5d7d8b4bcc from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d-predictor.18c67f1a9238c565 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 498925f3-199c-4275-a557-e2be07214d1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 55b3616a-5628-44bd-a781-1aadb703511f kind: Event lastTimestamp: "2026-07-28T15:54:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:59Z" name: success-200-isvc-ad35d-predictor.18c67f1e10af11c4 namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 0a42fc57-d09a-43bd-86b7-8ce35a42f512 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 55b3616a-5628-44bd-a781-1aadb703511f kind: Event lastTimestamp: "2026-07-28T15:54:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:54:59Z" name: success-200-isvc-ad35d-predictor.18c67f1e1177b57e namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 6597dce5-25c3-40d9-a0a2-2a04fee3f8ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: 55b3616a-5628-44bd-a781-1aadb703511f kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:14Z" name: success-200-isvc-ad35d-predictor.18c67f21906427f0 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: ae16e621-3f6a-4a24-bc16-5103268a08bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ad35d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: 55b3616a-5628-44bd-a781-1aadb703511f kind: Event lastTimestamp: "2026-07-28T15:55:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:55:14Z" name: success-200-isvc-ad35d-predictor.18c67f21918c52d4 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 561b9cf9-d818-4555-bb2b-45285d917776 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: c4d1e249-f686-43bc-9a03-b3f1db082079 kind: Event lastTimestamp: "2026-07-28T15:55:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:02Z" name: success-200-isvc-ad35d.18c67f1a9234adea namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: f5970dd8-cae4-40e5-8943-af8ec128d901 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: c4d1e249-f686-43bc-9a03-b3f1db082079 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: 'Failed to update status for InferenceService "success-200-isvc-ad35d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ad35d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d.18c67f1a9381c122 namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: 74aea66f-9ccc-4508-97cd-45e5d16c84c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: c4d1e249-f686-43bc-9a03-b3f1db082079 kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ad35d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:54:14Z" name: success-200-isvc-ad35d.18c67f1a938220bd namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 7469105b-29f6-4d47-be70-a861421aa1db reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:55:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ad35d namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: c4d1e249-f686-43bc-9a03-b3f1db082079 kind: Event lastTimestamp: "2026-07-28T15:55:02Z" message: InferenceService [success-200-isvc-ad35d] is Ready metadata: creationTimestamp: "2026-07-28T15:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:55:02Z" name: success-200-isvc-ad35d.18c67f25c6e4d9ce namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 15b0ad04-4d04-4056-b973-46e1c20b08e0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:04:39.728591Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25646" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b87a7-predictor-7865b79897-nxq5c to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac2ad3e75b namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 77a24f35-c45c-44b5-90d9-bcb3ba3c8e43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b87a7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac3ac5cfd8 namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: cb72e836-c21d-407c-8697-0843bc6ee032 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac68194900 namespace: kserve-ci-e2e-test resourceVersion: "25722" uid: 6bfca2e0-de20-45f0-aef4-ddcb5b22d13b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac69679bad namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: eefba2ff-5116-4eec-98ca-50ec88c8f985 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac6d7d392a namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 702fbbc6-fbf4-401e-ba4a-50f76f791736 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac6dee7edc namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: a4e67bb4-1c48-451a-9dc1-529ec3453879 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac6df98d29 namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 782ae490-c800-4c6c-9473-a91cbd05059e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac714ac8ee namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 84374fa4-29c9-454f-89e7-b32b6eb87df4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:04:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:04:40Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac71ba3898 namespace: kserve-ci-e2e-test resourceVersion: "25731" uid: 788c0008-bb90-4fef-84f2-fba0c6da9b6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:04:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:05:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:17Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fac82bd7955 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 6ad0af02-583f-4665-adad-d35e3d3e8784 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:05:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:49Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fbc85ffb5f6 namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: 8d360d54-8d7a-44ce-967a-1a257f8b00bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:05:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:49Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fbc8600460c namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: 1a85d3fa-a2ac-4e84-aebe-3af4fafcc391 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: d4bb668c-0cfe-49e7-9ddd-1c37b7458369 kind: Event lastTimestamp: "2026-07-28T16:05:52Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:52Z" name: success-200-isvc-b87a7-predictor-7865b79897-nxq5c.18c67fbd0ae6652b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 3d671fb5-82d5-4dd4-a1cb-95fdc52636bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b87a7-predictor-7865b79897 namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: ba401e05-dc7a-4c2c-bab3-bb715b3ff01a kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'Created pod: success-200-isvc-b87a7-predictor-7865b79897-nxq5c' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-b87a7-predictor-7865b79897.18c67fac2a3b8318 namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 933edfa6-76d4-4cfe-ab64-17026d6730b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: 5d8f67f7-8b89-4116-8c4e-4b35ce80477f kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: Scaled up replica set success-200-isvc-b87a7-predictor-7865b79897 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-b87a7-predictor.18c67fac266e5522 namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: 899a9960-a2fb-44ab-b906-e21877f2ba8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:04:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: bdd581b9-515f-4021-b017-8ef0c674de6b kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-b87a7-predictor.18c67fafa596c965 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: ae512e9b-45ae-4b40-9962-fc7fa04312c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:04:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: bdd581b9-515f-4021-b017-8ef0c674de6b kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:04:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-b87a7-predictor.18c67fafa5fff73e namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: c759003f-ac14-42d7-8fbc-20442b246356 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: bdd581b9-515f-4021-b017-8ef0c674de6b kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: success-200-isvc-b87a7-predictor.18c67fb6a41c0f86 namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: 64f7f041-44b2-4b59-a29e-1b9ec0fa2c13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b87a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: bdd581b9-515f-4021-b017-8ef0c674de6b kind: Event lastTimestamp: "2026-07-28T16:05:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:39Z" name: success-200-isvc-b87a7-predictor.18c67fb6a4876418 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: 19c1b97e-dc4e-4680-b9d1-6efba1e781a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 2d680d2d-eee7-433e-a178-abfa0a5efab0 kind: Event lastTimestamp: "2026-07-28T16:05:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:27Z" name: success-200-isvc-b87a7.18c67fac26c7993f namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: fd88d6d5-7228-4600-8a36-9ea0d511c397 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 2d680d2d-eee7-433e-a178-abfa0a5efab0 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'Failed to update status for InferenceService "success-200-isvc-b87a7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b87a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-b87a7.18c67fac276f7cbb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 5e1782e1-fdc4-4741-b4d4-2da66242e46e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 2d680d2d-eee7-433e-a178-abfa0a5efab0 kind: Event lastTimestamp: "2026-07-28T16:04:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b87a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:04:39Z" name: success-200-isvc-b87a7.18c67fac276fca94 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: 05136a47-f84c-48eb-bc2f-ae1b14b0fbeb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b87a7 namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: 2d680d2d-eee7-433e-a178-abfa0a5efab0 kind: Event lastTimestamp: "2026-07-28T16:05:27Z" message: InferenceService [success-200-isvc-b87a7] is Ready metadata: creationTimestamp: "2026-07-28T16:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:27Z" name: success-200-isvc-b87a7.18c67fb73eab3f11 namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: d24a4b1f-622e-47e7-8053-3d2ebb6a94b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:05:09.787501Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bd785-predictor-989fd784c-ndhgf to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb32a7a4f81 namespace: kserve-ci-e2e-test resourceVersion: "26038" uid: 679f5214-a28b-44e4-98a5-c9487609c320 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-bd785-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:10Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb334e9c702 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: 444235ee-0459-41ff-bcfb-2f724f304526 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb39d9858a5 namespace: kserve-ci-e2e-test resourceVersion: "26112" uid: 578aa0c6-a60a-47ad-b83e-ac06ccf0b3e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb39eee1fa5 namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: c7c0eae2-80f2-434d-9a68-149857941dab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb3a1ee3fd8 namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 2a2e22a3-2b75-4059-8196-6e427d332f1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb3a259a321 namespace: kserve-ci-e2e-test resourceVersion: "26118" uid: 93b1fb6c-0b8f-402b-b492-387f8f6a0004 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb3a26478a0 namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 74d94aa3-908e-4302-8cc0-f1d459ea5a7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb3a511ed90 namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: ab245cd9-d4dd-4aa1-afe6-d89941d3538b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:11Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb3a57ea838 namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: e46159e2-5bfe-4c95-ad59-7ecc06dac288 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:05:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:49Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c67fb3ff056bd2 namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: ef270f26-591c-4655-ad99-47f5b55b14b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:22:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:29Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c680a5396a0c2e namespace: kserve-ci-e2e-test resourceVersion: "33730" uid: c8e9f12e-8e88-4144-843d-0ddbd2a037d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bd785-predictor-989fd784c-ndhgf namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: bce3f050-9c6b-4a4f-a217-cbc4ab0fd4dd kind: Event lastTimestamp: "2026-07-28T16:22:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:29Z" name: success-200-isvc-bd785-predictor-989fd784c-ndhgf.18c680a5396b26d4 namespace: kserve-ci-e2e-test resourceVersion: "33731" uid: 9bbcd489-1512-4eb2-b8a5-267619bc8926 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bd785-predictor-989fd784c namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: f15a42e9-f496-4e82-a34b-5a95aaf2c3b7 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: 'Created pod: success-200-isvc-bd785-predictor-989fd784c-ndhgf' metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-bd785-predictor-989fd784c.18c67fb32a14e543 namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 6c3cb28c-73e7-4705-aee9-0d2ec9eca056 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: d2b78ca3-ccac-4916-a60a-778acf3dc8e8 kind: Event lastTimestamp: "2026-07-28T16:05:09Z" message: Scaled up replica set success-200-isvc-bd785-predictor-989fd784c from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:09Z" name: success-200-isvc-bd785-predictor.18c67fb32832ccb8 namespace: kserve-ci-e2e-test resourceVersion: "26015" uid: 0568b627-fbf5-4213-91f5-dad68e3bd717 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: f557a3ce-7a25-4e80-9f04-2973953d4e56 kind: Event lastTimestamp: "2026-07-28T16:05:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:24Z" name: success-200-isvc-bd785-predictor.18c67fb6a6b26e79 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: f794d0a0-1ece-4ef4-b204-44bfe28703d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: f557a3ce-7a25-4e80-9f04-2973953d4e56 kind: Event lastTimestamp: "2026-07-28T16:05:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:24Z" name: success-200-isvc-bd785-predictor.18c67fb6a720004b namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: b99af0a5-7f77-4c8f-abcc-7d5c95029bb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:05:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: f557a3ce-7a25-4e80-9f04-2973953d4e56 kind: Event lastTimestamp: "2026-07-28T16:06:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:09Z" name: success-200-isvc-bd785-predictor.18c67fba25edd6dd namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: c445a183-3a99-484d-bdb7-13929e948e7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:05:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bd785-predictor namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: f557a3ce-7a25-4e80-9f04-2973953d4e56 kind: Event lastTimestamp: "2026-07-28T16:06:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:09Z" name: success-200-isvc-bd785-predictor.18c67fba26c08bb9 namespace: kserve-ci-e2e-test resourceVersion: "26828" uid: a201ef40-051e-4f6a-96c7-763dfd2c7d67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bd785 namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: 4b879e2b-88ec-4215-9627-347d9bc1dc0c kind: Event lastTimestamp: "2026-07-28T16:05:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:05:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:59Z" name: success-200-isvc-bd785.18c67fb3286f576f namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 6481ad31-8ba2-47da-9ead-77d5eb9d9785 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: 4b879e2b-88ec-4215-9627-347d9bc1dc0c kind: Event lastTimestamp: "2026-07-28T16:05:59Z" message: InferenceService [success-200-isvc-bd785] is Ready metadata: creationTimestamp: "2026-07-28T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:59Z" name: success-200-isvc-bd785.18c67fbebb05b08b namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: c186d0f8-3fcf-45ad-b2bb-7824e97a4789 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:05:50.056975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 to ip-10-0-143-228.ec2.internal metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:05:50Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbc8ab9aef6 namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: aa53aef0-bd36-4ee1-85df-def1131d15c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-dd6ec-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:50Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbc93805a5b namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: dca316df-a6d4-458b-b573-b0f4aa2f7e3d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbcc840ec02 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: d70cda53-71bb-4d0e-8cc5-879de1fa8de8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbcc9b51314 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 30d8bae0-78f9-4f80-b95e-9c05ba843ae8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbccc993d42 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 3cedd442-df8e-4f21-9c2d-200c10e548a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbccd0b4a6b namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: 8d3c7082-2dc2-46e1-8621-bbed8a7acdec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbccd145643 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 1eb1390c-f307-4123-ad70-fe1f950ab988 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbcd03c5465 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: 18881422-bad0-4108-ad59-b206a39fc7dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:05:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:05:51Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbcd0a9b1c1 namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 9c13cd23-62c4-41cb-a170-00c9a17bceb2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:06:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:28Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c67fbd1e9c2138 namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 1a8e25c8-1951-40e1-9c9f-aea214dfa6b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:15:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:04Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c6803db5f3df5d namespace: kserve-ci-e2e-test resourceVersion: "30657" uid: 47df5c35-97f9-4275-8778-2cec115a4bb4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 575ece81-6bb8-4086-813f-d28b7c4eab37 kind: Event lastTimestamp: "2026-07-28T16:15:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:15:04Z" name: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7.18c6803db5f4851b namespace: kserve-ci-e2e-test resourceVersion: "30659" uid: 956280f3-080e-4dc1-96ce-9c9c2d63e35c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-228.ec2.internal source: component: kubelet host: ip-10-0-143-228.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dd6ec-predictor-6948f85895 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: a1171fe4-4c40-453d-9310-be85a552bc84 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'Created pod: success-200-isvc-dd6ec-predictor-6948f85895-wp4q7' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:50Z" name: success-200-isvc-dd6ec-predictor-6948f85895.18c67fbc8a39b3ef namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 7aa4b1fa-6deb-4984-9241-d159775b1fa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 33f672e5-c902-4fdc-9a43-3bda74e44c8b kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: Scaled up replica set success-200-isvc-dd6ec-predictor-6948f85895 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:05:50Z" name: success-200-isvc-dd6ec-predictor.18c67fbc88740022 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 83185ffd-d2c8-439f-8cac-5560c888c47d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 97e135d4-84c6-4d27-98b7-7985b9de3326 kind: Event lastTimestamp: "2026-07-28T16:06:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:20Z" name: success-200-isvc-dd6ec-predictor.18c67fc006e54df0 namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: 7d83d242-3385-4e13-a7cb-2ffc2b18c36f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 97e135d4-84c6-4d27-98b7-7985b9de3326 kind: Event lastTimestamp: "2026-07-28T16:06:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:20Z" name: success-200-isvc-dd6ec-predictor.18c67fc0074c055e namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 2167f3bc-4460-465e-822c-a8d8cbcc2d9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:06:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: 97e135d4-84c6-4d27-98b7-7985b9de3326 kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:50Z" name: success-200-isvc-dd6ec-predictor.18c67fc7052e94b8 namespace: kserve-ci-e2e-test resourceVersion: "27193" uid: 20df6913-9a40-4653-ae14-fe2262c8ddb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:06:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd6ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: 97e135d4-84c6-4d27-98b7-7985b9de3326 kind: Event lastTimestamp: "2026-07-28T16:06:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:50Z" name: success-200-isvc-dd6ec-predictor.18c67fc705a872d0 namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: 0a3bc0ee-aaf5-4638-871d-82bd5da3ae1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: fe8e574a-c39c-4e1d-ac04-42f4e0eb2515 kind: Event lastTimestamp: "2026-07-28T16:06:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:06:38Z" name: success-200-isvc-dd6ec.18c67fbc888c2cbd namespace: kserve-ci-e2e-test resourceVersion: "27100" uid: 30ec26c0-fd1c-4a40-acdc-71cd69acb63a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: fe8e574a-c39c-4e1d-ac04-42f4e0eb2515 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-dd6ec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dd6ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:50Z" name: success-200-isvc-dd6ec.18c67fbc89763829 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 1f2f7ce8-2a22-4f74-821b-23b430ce315e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:05:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: fe8e574a-c39c-4e1d-ac04-42f4e0eb2515 kind: Event lastTimestamp: "2026-07-28T16:05:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dd6ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:05:50Z" name: success-200-isvc-dd6ec.18c67fbc8976aa53 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 10fabe07-4e31-4f44-9da9-28048ab83459 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd6ec namespace: kserve-ci-e2e-test resourceVersion: "27098" uid: fe8e574a-c39c-4e1d-ac04-42f4e0eb2515 kind: Event lastTimestamp: "2026-07-28T16:06:38Z" message: InferenceService [success-200-isvc-dd6ec] is Ready metadata: creationTimestamp: "2026-07-28T16:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:06:38Z" name: success-200-isvc-dd6ec.18c67fc7daceec04 namespace: kserve-ci-e2e-test resourceVersion: "27099" uid: 756141aa-78b8-47ae-9f22-5472a2cc7ee7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:45:49.558404Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4b277-54cd4bc6c7-w7b2h to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:45:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:45:49Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67ea5076f372a namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 565f777a-b2f6-4d52-8a2f-897b3e4ec28b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:45:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4b277-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:49Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67ea511195aff namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 0ec04fa5-622d-482a-a296-b9c4886e47f5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:45:50Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:45:50Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67ea544bba8fd namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 1cfd1c35-20e6-415f-8836-9eebf4d030d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4b277} kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:45:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T15:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:50Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67ea546333b79 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 5f54a40a-8246-480e-8dda-50317eac6c1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4b277} kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:45:50Z" message: 'Created container: switch-graph-4b277' metadata: creationTimestamp: "2026-07-28T15:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:50Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67ea5492d728f namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 0b935851-18f1-4f7e-ba47-86315ec58b66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4b277} kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:45:50Z" message: Started container switch-graph-4b277 metadata: creationTimestamp: "2026-07-28T15:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:50Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67ea5499beac1 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: 61ad7edb-0dc2-4f1c-891d-ada182aa0a55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4b277} kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:54:14Z" message: Stopping container switch-graph-4b277 metadata: creationTimestamp: "2026-07-28T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:14Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67f1a89578640 namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: e24060dd-8e27-431f-a0a3-1aedb243a1f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4b277} kind: Pod name: switch-graph-4b277-54cd4bc6c7-w7b2h namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 43e5bece-01b6-40cf-a368-c1278d0afa4e kind: Event lastTimestamp: "2026-07-28T15:54:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T15:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:54:42Z" name: switch-graph-4b277-54cd4bc6c7-w7b2h.18c67f1b4d2255ab namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 9c6d1056-4ef8-4b05-90e9-8eb9cba2d95f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4b277-54cd4bc6c7 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 92da9517-3f1d-493c-9834-1b434c7ede0b kind: Event lastTimestamp: "2026-07-28T15:45:49Z" message: 'Created pod: switch-graph-4b277-54cd4bc6c7-w7b2h' metadata: creationTimestamp: "2026-07-28T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:49Z" name: switch-graph-4b277-54cd4bc6c7.18c67ea506f21683 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 44be5292-c9f5-4a8c-9fb8-bc96c7228495 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4b277 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 440fc45a-ccd7-4ec3-a423-2a0cf13fa5bd kind: Event lastTimestamp: "2026-07-28T15:45:49Z" message: Scaled up replica set switch-graph-4b277-54cd4bc6c7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:45:49Z" name: switch-graph-4b277.18c67ea50604e518 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: e0afe994-1799-46de-8ede-ef26cab790c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:45:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4b277 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 7daf22d8-7ded-46ca-97e3-d0c0e2db3101 kind: Event lastTimestamp: "2026-07-28T15:45:49Z" message: InferenceGraph [switch-graph-4b277] is Ready metadata: creationTimestamp: "2026-07-28T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:45:49Z" name: switch-graph-4b277.18c67ea50a2c3b6d namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 7d1f14e4-aa09-43fa-b539-d53789d60e4d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4b277 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: e275a152-0e02-4495-969e-0c9a096b6fdf kind: Event lastTimestamp: "2026-07-28T15:46:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:19Z" name: switch-graph-4b277.18c67ea8846e102d namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 5ba96015-1902-4e80-a999-063237a8bd80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T15:46:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4b277 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: e275a152-0e02-4495-969e-0c9a096b6fdf kind: Event lastTimestamp: "2026-07-28T15:46:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T15:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:46:19Z" name: switch-graph-4b277.18c67ea884c39099 namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 2aec7115-264d-4b8b-9b15-0cc5055a1a16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T15:44:39.171000Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8fc0c-b5cb6cc48-tdw5s to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T15:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T15:44:39Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e94a404a007 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 8297388a-bfaa-443d-ac8e-13e7f29b3f2d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T15:44:39Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e94bd1481b9 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 6c156500-90f8-4ded-9963-c5e4310fd0cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fc0c} kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" metadata: creationTimestamp: "2026-07-28T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:39Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e94be6012c8 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 156dd412-de35-4f9d-bb70-9a5cc8b626a4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fc0c} kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:44:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" in 2.145s (2.145s including waiting). Image size: 218226816 bytes.' metadata: creationTimestamp: "2026-07-28T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:41Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e953e46750d namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 15f5d918-6176-4f58-b1ec-05aef902d844 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fc0c} kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:44:41Z" message: 'Created container: switch-graph-8fc0c' metadata: creationTimestamp: "2026-07-28T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:41Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e954183b7f3 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 2d07fbe9-8594-41d6-9d91-3014dd3f5623 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fc0c} kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:44:41Z" message: Started container switch-graph-8fc0c metadata: creationTimestamp: "2026-07-28T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:41Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e9541e64cb7 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 1032a40b-ad44-4c03-a596-f79d11c16df0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fc0c} kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:44:49Z" message: Stopping container switch-graph-8fc0c metadata: creationTimestamp: "2026-07-28T15:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:44:49Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e96ff2ae903 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: 01c213fd-cbf7-4c87-ba38-9f33e61cc81d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T15:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fc0c} kind: Pod name: switch-graph-8fc0c-b5cb6cc48-tdw5s namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 72c77117-f53a-48db-9b2a-fd649c404509 kind: Event lastTimestamp: "2026-07-28T15:45:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T15:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T15:45:18Z" name: switch-graph-8fc0c-b5cb6cc48-tdw5s.18c67e97eb23e182 namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 30c5f60b-ab30-4cde-9f7a-ca5b0c2825b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8fc0c-b5cb6cc48 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 9792dcd8-d5ee-4279-8fcb-ef4f371f01bb kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: 'Created pod: switch-graph-8fc0c-b5cb6cc48-tdw5s' metadata: creationTimestamp: "2026-07-28T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:39Z" name: switch-graph-8fc0c-b5cb6cc48.18c67e94a3949612 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 13453c1b-d0f1-465f-8abe-17e0ce8a64dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8fc0c namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: ec6ec658-3146-4e16-838e-b1fb69c6c46d kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: Scaled up replica set switch-graph-8fc0c-b5cb6cc48 from 0 to 1 metadata: creationTimestamp: "2026-07-28T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T15:44:39Z" name: switch-graph-8fc0c.18c67e94a2a1881a namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: cc32fff1-53a6-467d-812c-2f0d33dc1929 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T15:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8fc0c namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 290cc543-751c-4279-adf2-eb42728e242b kind: Event lastTimestamp: "2026-07-28T15:44:39Z" message: InferenceGraph [switch-graph-8fc0c] is Ready metadata: creationTimestamp: "2026-07-28T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T15:44:39Z" name: switch-graph-8fc0c.18c67e94a6d3704e namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: dc093922-8b9d-48fa-95cd-6ff026670103 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:06:09.868650Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-bd785-64486d8c7-99n55 to ip-10-0-134-118.ec2.internal metadata: creationTimestamp: "2026-07-28T16:06:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:06:09Z" name: switch-graph-bd785-64486d8c7-99n55.18c67fc12798a96d namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: 0eef8d72-a062-41c6-b6de-f9db95ab71c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c5f6dfdb5-bg7rc source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:06:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:06:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-bd785-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:10Z" name: switch-graph-bd785-64486d8c7-99n55.18c67fc1300e21ff namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: 0acef27c-430c-4730-971f-f3ce611b24a5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:06:11Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:06:11Z" name: switch-graph-bd785-64486d8c7-99n55.18c67fc19a8e5b83 namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 0b3e9c00-40c4-4695-9231-ceb8b847fd25 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bd785} kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:06:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:40920ff6a00036cf8932ff07962738516dfac83c34c2855a78a35a4531f74d96" already present on machine metadata: creationTimestamp: "2026-07-28T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:11Z" name: switch-graph-bd785-64486d8c7-99n55.18c67fc19bcd0720 namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: 6544c010-3aca-47f7-8d22-30f290ee473b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bd785} kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:06:11Z" message: 'Created container: switch-graph-bd785' metadata: creationTimestamp: "2026-07-28T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:11Z" name: switch-graph-bd785-64486d8c7-99n55.18c67fc19ed9f3b8 namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 75c59f86-196c-4a99-be35-4d85611977c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bd785} kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:06:11Z" message: Started container switch-graph-bd785 metadata: creationTimestamp: "2026-07-28T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:06:11Z" name: switch-graph-bd785-64486d8c7-99n55.18c67fc19f4a789e namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: 08f04445-4d39-4e81-a3bb-e71a029d45bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bd785} kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:22:29Z" message: Stopping container switch-graph-bd785 metadata: creationTimestamp: "2026-07-28T16:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:29Z" name: switch-graph-bd785-64486d8c7-99n55.18c680a5296705fa namespace: kserve-ci-e2e-test resourceVersion: "33711" uid: 68af39ef-c18d-4637-8af3-393741b3381d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bd785} kind: Pod name: switch-graph-bd785-64486d8c7-99n55 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 71c375ee-d13e-4b2c-9261-c0074d366f20 kind: Event lastTimestamp: "2026-07-28T16:22:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:58Z" name: switch-graph-bd785-64486d8c7-99n55.18c680a613b8560f namespace: kserve-ci-e2e-test resourceVersion: "33989" uid: 7d0d2055-4084-40f8-abe8-ec9955ca9885 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-118.ec2.internal source: component: kubelet host: ip-10-0-134-118.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-bd785-64486d8c7 namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 0aa7685f-3f61-4792-a026-26473025d565 kind: Event lastTimestamp: "2026-07-28T16:06:09Z" message: 'Created pod: switch-graph-bd785-64486d8c7-99n55' metadata: creationTimestamp: "2026-07-28T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:09Z" name: switch-graph-bd785-64486d8c7.18c67fc127207001 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 95b8fe91-ba1d-45fc-9955-566f03a8e2c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: a718043f-c10c-4bd2-9df0-728553bf6f89 kind: Event lastTimestamp: "2026-07-28T16:06:09Z" message: Scaled up replica set switch-graph-bd785-64486d8c7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:09Z" name: switch-graph-bd785.18c67fc126453004 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: dd608b9f-d050-4956-8c1a-a9b0dc9d7e46 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 96fe7d71-d66f-4ef7-b662-fb257100dc8b kind: Event lastTimestamp: "2026-07-28T16:06:09Z" message: InferenceGraph [switch-graph-bd785] is Ready metadata: creationTimestamp: "2026-07-28T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:06:09Z" name: switch-graph-bd785.18c67fc12ac8c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 7847db33-3977-49c5-8e7e-bdb6f35ce653 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: bbcd6f25-edd5-40b6-9810-2610a745f25f kind: Event lastTimestamp: "2026-07-28T16:06:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:24Z" name: switch-graph-bd785.18c67fc4a4c47ade namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: c3eba5ff-8f7b-4461-8147-573a5ee58589 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: bbcd6f25-edd5-40b6-9810-2610a745f25f kind: Event lastTimestamp: "2026-07-28T16:06:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:24Z" name: switch-graph-bd785.18c67fc4a528f8e9 namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 0eb822d2-8cc7-4606-b703-5403425c5a59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: bbcd6f25-edd5-40b6-9810-2610a745f25f kind: Event lastTimestamp: "2026-07-28T16:06:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:39Z" name: switch-graph-bd785.18c67fc825063522 namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: 68fea20c-2bbe-44c8-8f36-71a357079099 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:06:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bd785 namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: bbcd6f25-edd5-40b6-9810-2610a745f25f kind: Event lastTimestamp: "2026-07-28T16:06:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:06:39Z" name: switch-graph-bd785.18c67fc82589187c namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: cab9bc85-d9f3-4d7f-86ea-b60af4c7ddf8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33990"