--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:02:06.466149Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-14ef6-8648bc46f9-f667j to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b10e1440b441 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 30c44718-162b-4c96-b0fa-8b1c34739a69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b10e2efec189 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: c75ba035-f37d-4696-b567-c447e0f1b909 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-14ef6} kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b10e30655d57 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: 064dedf4-1434-49f1-bb22-f4c718698666 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-14ef6} kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'Created container: ensemble-graph-14ef6' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b10e33e53791 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: a9efb04c-5e4d-4581-a6b5-28fd649561b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-14ef6} kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Started container ensemble-graph-14ef6 metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b10e3464009b namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 309cb502-858b-4229-9d86-c6776ee59f3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-14ef6} kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: "2026-07-09T18:10:20Z" message: Stopping container ensemble-graph-14ef6 metadata: creationTimestamp: "2026-07-09T18:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:20Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b18134d1c15f namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: 74b8498c-9dcc-4ccb-a0ec-cda505ea220b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-14ef6} kind: Pod name: ensemble-graph-14ef6-8648bc46f9-f667j namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: eacfc064-afd8-4401-9397-02488702a206 kind: Event lastTimestamp: "2026-07-09T18:10:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:48Z" name: ensemble-graph-14ef6-8648bc46f9-f667j.18c0b181b6018581 namespace: kserve-ci-e2e-test resourceVersion: "25273" uid: 7631383b-7b42-4055-85d2-82c42adad5b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-14ef6-8648bc46f9 namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 8f472e21-b8da-470a-b3d4-dcad514b613d kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'Created pod: ensemble-graph-14ef6-8648bc46f9-f667j' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6-8648bc46f9.18c0b10e14b3dc69 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 42313529-e0ef-450c-8f3a-251ef77de611 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 5478be91-c715-431a-8636-92fd41eb13d5 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Scaled up replica set ensemble-graph-14ef6-8648bc46f9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6.18c0b10e104a0053 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: abe37510-9db2-4ca4-a004-a1d0c6e29aa5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 86ebcce4-f4da-4bab-92a4-15fb42781f0a kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: InferenceGraph [ensemble-graph-14ef6] is Ready metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:06Z" name: ensemble-graph-14ef6.18c0b10e20682786 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 8bae5ce4-52f6-471e-afbd-b61a0ab323ba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 1a101204-6c5b-4d81-b925-9ac492a85919 kind: Event lastTimestamp: "2026-07-09T18:02:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:36Z" name: ensemble-graph-14ef6.18c0b1118eeeafdb namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 42d508cb-d028-4cab-9778-bd2d93fe8697 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 1a101204-6c5b-4d81-b925-9ac492a85919 kind: Event lastTimestamp: "2026-07-09T18:02:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:36Z" name: ensemble-graph-14ef6.18c0b1118f62f828 namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: c9e51630-b1fd-405d-8d41-7ed48c930ad2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:00:56.121644Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5bd2e-845cdb685d-fmrbj to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:00:56Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b0fdb364c1b7 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 088c8fb5-ac74-4026-8490-c76e7fb9035a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-5bd2e-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b0fdc0129bfc namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 99147eaf-96b0-489f-b587-14fb47ccad80 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:00:57Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b0fdf238bb14 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 361c3708-c11a-4b26-a5a9-418f2a16db38 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5bd2e} kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b0fdf3e1e31c namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: f0c403a1-4ce7-45d0-ba4f-d7d201ffe322 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5bd2e} kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: 'Created container: ensemble-graph-5bd2e' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b0fdf6bf46de namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 692e4d80-08f5-4bc8-8f38-a55d6c817267 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5bd2e} kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Started container ensemble-graph-5bd2e metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b0fdf73ab06a namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 74d8435f-30be-4cb3-9a62-9584525e3b65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5bd2e} kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Stopping container ensemble-graph-5bd2e metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b1000b4b8822 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: babaa834-6eb2-4469-8796-76735cefdb00 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5bd2e} kind: Pod name: ensemble-graph-5bd2e-845cdb685d-fmrbj namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: b46fd1cc-3acd-47c6-a5b4-7cbeb4caaed4 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:33Z" name: ensemble-graph-5bd2e-845cdb685d-fmrbj.18c0b100a803de88 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 57b0b7e0-56af-436f-bb09-99a8851408b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5bd2e-845cdb685d namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 6be03457-9354-418f-b3a8-f4a738443f4c kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Created pod: ensemble-graph-5bd2e-845cdb685d-fmrbj' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: ensemble-graph-5bd2e-845cdb685d.18c0b0fdb2e18b5b namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: caf8d345-fe58-4cec-ae6e-62183f682f4e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 19f22613-5c0f-4946-963c-a5f7e21328ca kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Scaled up replica set ensemble-graph-5bd2e-845cdb685d from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: ensemble-graph-5bd2e.18c0b0fdb1d06661 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 79a3a8ff-56b3-4ceb-a3d7-50ea1652bb85 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 68db9c75-11ff-4675-bef3-40f9d5ba3d20 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: InferenceGraph [ensemble-graph-5bd2e] is Ready metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:56Z" name: ensemble-graph-5bd2e.18c0b0fdb727ee8b namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 11779d8a-f52e-402f-be7b-846cf8a49e2a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:10:21.273063Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25020" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b181490b088f namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: f552c1d2-3f37-4ff8-9c94-2c377d46c01f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b18163ba8906 namespace: kserve-ci-e2e-test resourceVersion: "25045" uid: f2084d69-ba43-4701-b008-8c82021f5d02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b18165c62b7b namespace: kserve-ci-e2e-test resourceVersion: "25048" uid: f4683d1f-1098-42c4-95f6-9ba5d005529f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b1816817c698 namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: 906d8f3e-980f-4be3-9c34-d5d806064e59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b181688d8d73 namespace: kserve-ci-e2e-test resourceVersion: "25052" uid: 382c80a9-da72-4a10-b6e6-7ac11cc9bd96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b1816896f54d namespace: kserve-ci-e2e-test resourceVersion: "25053" uid: 4f29b85a-6a00-4d3f-a9a3-3737cda2ca79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b1816ba35eab namespace: kserve-ci-e2e-test resourceVersion: "25055" uid: c5304154-22aa-4fb9-a7c6-780130f096d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b1816c2dca7a namespace: kserve-ci-e2e-test resourceVersion: "25056" uid: efa4c647-b0bd-4aa3-99aa-0e8041b49d64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:10:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:59Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b181af80ba0d namespace: kserve-ci-e2e-test resourceVersion: "25354" uid: fa755483-23d9-4655-95d5-d72c2498076e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b191a27796a6 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 4734e0d7-1b4f-4a6b-b8b5-f682e1c73a5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b191a2789412 namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 59c95fb5-8a71-4cf7-991e-de69b2fd67a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 4857bdc5-980d-47e0-aff9-fdd41039776b kind: Event lastTimestamp: "2026-07-09T18:11:33Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:34Z" name: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2.18c0b19237cbb274 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: d62c7cec-475d-498f-8c80-7702f78d1a22 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-07956-predictor-7d74d5c7dc namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: fc535111-557b-4ab3-8918-55fccaf5e584 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: 'Created pod: error-404-isvc-07956-predictor-7d74d5c7dc-zctn2' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor-7d74d5c7dc.18c0b18148a1739c namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: 025a5f44-c9bf-4633-b1d1-340fa336b344 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: 9c47318e-4c03-4356-91b3-a00b94fbe100 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Scaled up replica set error-404-isvc-07956-predictor-7d74d5c7dc from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-07956-predictor.18c0b18146c3f36a namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: ef05bd0a-741d-45d0-8ba2-e73ab0250084 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: e20d7fc7-3339-40fe-b6b6-d11b912a2046 kind: Event lastTimestamp: "2026-07-09T18:10:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:36Z" name: error-404-isvc-07956-predictor.18c0b184c5854fbb namespace: kserve-ci-e2e-test resourceVersion: "25183" uid: a3e9519b-8d4a-4ed6-b485-741601dcd49c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: e20d7fc7-3339-40fe-b6b6-d11b912a2046 kind: Event lastTimestamp: "2026-07-09T18:10:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:36Z" name: error-404-isvc-07956-predictor.18c0b184c5c19c51 namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: e1b3967c-f397-41c9-b094-b59f3d818d35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:10:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25184" uid: e20d7fc7-3339-40fe-b6b6-d11b912a2046 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-07956-predictor.18c0b188447a6cec namespace: kserve-ci-e2e-test resourceVersion: "25622" uid: 19a076b5-52ca-4b41-a54d-593bd3014f19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:10:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25184" uid: e20d7fc7-3339-40fe-b6b6-d11b912a2046 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-07956-predictor.18c0b18844ec9fb4 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: abdd4e85-c0fb-46ce-ae54-38671510a942 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-07956 namespace: kserve-ci-e2e-test resourceVersion: "24994" uid: e30dfbfe-5afa-498f-a230-cecf342b8d5b kind: Event lastTimestamp: "2026-07-09T18:11:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:09Z" name: error-404-isvc-07956.18c0b18147085f94 namespace: kserve-ci-e2e-test resourceVersion: "25450" uid: c0c9f8db-fd90-48bf-852b-270d168d3f14 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-07956 namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: e30dfbfe-5afa-498f-a230-cecf342b8d5b kind: Event lastTimestamp: "2026-07-09T18:11:09Z" message: InferenceService [error-404-isvc-07956] is Ready metadata: creationTimestamp: "2026-07-09T18:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:09Z" name: error-404-isvc-07956.18c0b18c6b1b40f0 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 68ace589-00af-42a0-b291-422400678568 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:01:06.504225Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-14ef6-predictor-6478f88585-pt8tz to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b1001e3e54cb namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 542e3c55-80c4-4173-95e7-fae3623e27f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b1003802c6ce namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: a2598c66-cb9a-40ab-a975-ba588bffc155 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b100395f5ad2 namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 36dfe1db-1351-463d-89f3-7f6db9be7dfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b1003d07f13f namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: fa13ab2e-5a10-453c-ad50-95534366c658 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b1003d77a942 namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 21098514-0403-4f64-945b-7c4b965a170e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b1003d81cda0 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 5e8d0379-0490-4247-8bb2-b9932837dac6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b100417d01cb namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 401124d7-1d4f-473e-b024-25831505dd58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b10041eaf3e9 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: c4f6d543-efbb-4791-acb6-06aba1b77e20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:10:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:23Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b100579ea174 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: ee9d4974-32e5-466e-82fb-2f0f39c9f005 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b181410f75af namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: c83924b0-8c11-4b22-857d-08e3f52daec3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b181410fe77d namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 9aca493f-313c-4c46-be73-1006aa5366bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 1326a459-a070-4137-9574-a50ba77b6ef9 kind: Event lastTimestamp: "2026-07-09T18:10:23Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:23Z" name: error-404-isvc-14ef6-predictor-6478f88585-pt8tz.18c0b181cc4fe837 namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: a6f7600d-8359-42b2-8844-c9faa36c6d91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-14ef6-predictor-6478f88585 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: 1d245d11-bfcb-4112-abdb-712d9920c5fe kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'Created pod: error-404-isvc-14ef6-predictor-6478f88585-pt8tz' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6-predictor-6478f88585.18c0b1001d2a45e6 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: ace8d07f-8140-4802-9a7e-9f4207161c32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: bbf05c20-8c92-4811-8797-a5d90ef91745 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Scaled up replica set error-404-isvc-14ef6-predictor-6478f88585 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6-predictor.18c0b10019c52e55 namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 709e9943-86e2-4e52-8473-1a767ade8e9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: eabadbc7-5e49-4f8e-8ccf-9819e3dc1d68 kind: Event lastTimestamp: "2026-07-09T18:01:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:36Z" name: error-404-isvc-14ef6-predictor.18c0b1039787906b namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: 73c06b2d-d2fc-4bef-bc18-1abd9fae60b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: eabadbc7-5e49-4f8e-8ccf-9819e3dc1d68 kind: Event lastTimestamp: "2026-07-09T18:01:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:36Z" name: error-404-isvc-14ef6-predictor.18c0b103980544c8 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 67c89daf-7cfc-41a9-94f3-4ae4c1dc7ef9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: eabadbc7-5e49-4f8e-8ccf-9819e3dc1d68 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-14ef6-predictor.18c0b10a9568cb6d namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: 30895348-75fb-42a7-a5a4-9df516f8a594 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: eabadbc7-5e49-4f8e-8ccf-9819e3dc1d68 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-14ef6-predictor.18c0b10a95ca2e32 namespace: kserve-ci-e2e-test resourceVersion: "21301" uid: c948e4fc-eaf1-4c9a-9e27-99635268d00b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: d8d0e9d6-e6c6-469e-a1f8-0fe8aa4447eb kind: Event lastTimestamp: "2026-07-09T18:01:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:53Z" name: error-404-isvc-14ef6.18c0b100184bf073 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 213aea86-c7a5-408b-ab6e-10e9b13bc09e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: d8d0e9d6-e6c6-469e-a1f8-0fe8aa4447eb kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-14ef6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-14ef6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6.18c0b10018ff8df6 namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 8e1508b9-f9a4-46e5-9a93-cf0c7adecad5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: d8d0e9d6-e6c6-469e-a1f8-0fe8aa4447eb kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-14ef6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-14ef6.18c0b10018fff398 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: fb43fd3e-cca3-46d2-98c7-323f314197c5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: d8d0e9d6-e6c6-469e-a1f8-0fe8aa4447eb kind: Event lastTimestamp: "2026-07-09T18:01:53Z" message: InferenceService [error-404-isvc-14ef6] is Ready metadata: creationTimestamp: "2026-07-09T18:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:53Z" name: error-404-isvc-14ef6.18c0b10b136c6381 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 91f9f3ea-65b7-4f80-92aa-8bf928312a59 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:00:56.280345Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-53a2e-predictor-5ccf567585-c458v to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fdbcda5287 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: a8a3d768-ba2f-4935-951a-c386fcc8f1bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fdd7b5f269 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: ba2231e0-90ff-4df6-86bb-6ca2baf477ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fdd95c4db0 namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: 11fa1aa8-29ba-4cab-af0b-fed96eec49db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fddc630612 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 70617bc1-bc81-4909-b0e1-a0d40d9b241e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fddccf75c2 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: bfd1a0e0-97ef-4d62-ae3a-0958ef53a1c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fddcd8f24d namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 5b4caf2c-3da3-4f70-a5a1-6f8d04e5160d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fde153a144 namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: aa55c5a1-7d45-4b62-94f6-907898f89641 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fde1c3e0b8 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 29948ec7-d02f-434b-ba0f-eacdac7e6d43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:33Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b0fe0026d55c namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: a0a6966d-0367-4956-b5cd-93d97a91872d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b10e193402c0 namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: 30122879-88c6-404a-8a15-431f37c58ea9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b10e19348e34 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: e15b71bf-1526-4b4d-8419-103173351f98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a2e-predictor-5ccf567585-c458v namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 1ad27de7-ff28-4b41-bce2-2552715cd5b4 kind: Event lastTimestamp: "2026-07-09T18:02:08Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:08Z" name: error-404-isvc-53a2e-predictor-5ccf567585-c458v.18c0b10e88650059 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 56528eea-0bf8-4c2e-ad17-411e1dd65d87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-53a2e-predictor-5ccf567585 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 55097bb8-b51b-4242-9614-697dbe274383 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Created pod: error-404-isvc-53a2e-predictor-5ccf567585-c458v' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor-5ccf567585.18c0b0fdbc6c91dd namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 1e4e5807-3a8a-48f1-bcdd-970db8e0fbff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 4bfeb1ab-1110-47e9-a1f4-0e3e76d9208c kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Scaled up replica set error-404-isvc-53a2e-predictor-5ccf567585 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e-predictor.18c0b0fdba8cbd22 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 0033af43-d657-4a5f-952e-37bfd51772e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: e0915948-d82d-49ed-82f9-49c94e6a09f7 kind: Event lastTimestamp: "2026-07-09T18:01:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:26Z" name: error-404-isvc-53a2e-predictor.18c0b1013939e840 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c6d7157c-f469-4aa9-a2a9-58e1947e97ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: e0915948-d82d-49ed-82f9-49c94e6a09f7 kind: Event lastTimestamp: "2026-07-09T18:01:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:26Z" name: error-404-isvc-53a2e-predictor.18c0b1013981dcc0 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 95f1d0f6-c8c0-4ac2-85f6-c7f4f29c587b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: e0915948-d82d-49ed-82f9-49c94e6a09f7 kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:56Z" name: error-404-isvc-53a2e-predictor.18c0b10836e28690 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 965cc60e-03bf-4cfd-b38e-b03d8e2c7151 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: e0915948-d82d-49ed-82f9-49c94e6a09f7 kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:56Z" name: error-404-isvc-53a2e-predictor.18c0b108375556f0 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: a30ccc13-b61c-4dec-9238-7ac49c433c7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 99d5ec6f-2c5d-4be8-a65a-c9c866898767 kind: Event lastTimestamp: "2026-07-09T18:01:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:43Z" name: error-404-isvc-53a2e.18c0b0fdbab7c2fe namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: aaa4aa25-e91d-418a-baca-3180452fcfd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 99d5ec6f-2c5d-4be8-a65a-c9c866898767 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-53a2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-53a2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e.18c0b0fdbb41a94b namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: bfbd46f4-b287-4561-b986-24f253413e64 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 99d5ec6f-2c5d-4be8-a65a-c9c866898767 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-53a2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-53a2e.18c0b0fdbb41f1d8 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 0e1d928b-fbc9-4fb3-9791-75651b2b0f71 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 99d5ec6f-2c5d-4be8-a65a-c9c866898767 kind: Event lastTimestamp: "2026-07-09T18:01:43Z" message: InferenceService [error-404-isvc-53a2e] is Ready metadata: creationTimestamp: "2026-07-09T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:43Z" name: error-404-isvc-53a2e.18c0b108bf3c2288 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 243612ed-2bca-4b5e-9f90-694a234c7285 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:11:31.516013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25889" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191a3d95310 namespace: kserve-ci-e2e-test resourceVersion: "25894" uid: dc108868-43ca-430b-bd09-1f7bb9fda6ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-55ba8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191b06996d3 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: f056511a-f16f-4019-b9fa-c4966e4b285c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25898" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191e28814a8 namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: a27f9640-9167-4de1-a63d-a03038bd5177 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191e40a2441 namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: 9e889389-1403-4a79-8609-2fbb4cf46b46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191e74cdefd namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 1f22dd42-d645-4936-ad61-c6152d7ada3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191e7ea87ca namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: e4bedb58-c426-48ba-9b5a-d6b5c1ff7aef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191e7f3a51e namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 0092f2c9-25ac-4764-b8b7-30e86da09033 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191ec52e522 namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 1bf03399-9be3-4fcb-9f3b-38a47c440118 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b191ecc8818e namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 7d15677c-3426-427b-aed7-83fdf000d957 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:12:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:19Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b1921143727a namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: dd6b7c4c-72bd-4eff-b275-39d8c2979382 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:20:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:20:46Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b212c520098e namespace: kserve-ci-e2e-test resourceVersion: "30349" uid: 3fba4564-eb68-4c0d-819a-6fcb83be045b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 811379ba-7a26-4cc2-ac2c-89d73fb5d560 kind: Event lastTimestamp: "2026-07-09T18:20:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:20:46Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg.18c0b212c52083f7 namespace: kserve-ci-e2e-test resourceVersion: "30350" uid: fa2b46f1-053d-4976-83d5-26d5b92929d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-55ba8-predictor-7c8f5d49f9 namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 3c40e12a-7e19-4a1a-bdb6-136991cb210a kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'Created pod: error-404-isvc-55ba8-predictor-7c8f5d49f9-wmxwg' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-55ba8-predictor-7c8f5d49f9.18c0b191a35459c2 namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: b64691e0-88ed-4958-8b7f-d3c27b178932 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: 982e9c0e-cd42-42b0-8d95-b7446abdb5f8 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Scaled up replica set error-404-isvc-55ba8-predictor-7c8f5d49f9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-55ba8-predictor.18c0b191a1aa869d namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: bbe9f925-173c-4592-b350-c482735f5e9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: cc644e92-2d37-4794-8dce-70443656f1f8 kind: Event lastTimestamp: "2026-07-09T18:12:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:01Z" name: error-404-isvc-55ba8-predictor.18c0b19520254269 namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: fa9894e5-71cc-4460-a141-ff65b65cafda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: cc644e92-2d37-4794-8dce-70443656f1f8 kind: Event lastTimestamp: "2026-07-09T18:12:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:01Z" name: error-404-isvc-55ba8-predictor.18c0b195208c5028 namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: 8ba82868-8485-412a-b525-650816c8e51d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:12:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: cc644e92-2d37-4794-8dce-70443656f1f8 kind: Event lastTimestamp: "2026-07-09T18:12:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:46Z" name: error-404-isvc-55ba8-predictor.18c0b19c1deed47a namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: 24a9a8e9-10e9-4b3a-8249-b0b8218242f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:12:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: cc644e92-2d37-4794-8dce-70443656f1f8 kind: Event lastTimestamp: "2026-07-09T18:12:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:46Z" name: error-404-isvc-55ba8-predictor.18c0b19c1e54b3a6 namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: f4404728-4d31-440a-bb59-072a700c1f0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: ea78178f-4d40-476f-853d-8493e194cbd8 kind: Event lastTimestamp: "2026-07-09T18:12:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:29Z" name: error-404-isvc-55ba8.18c0b191a1be6f1b namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 6870d632-7561-40d5-8ce0-1c0395dcc5e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: ea78178f-4d40-476f-853d-8493e194cbd8 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-55ba8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-55ba8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-55ba8.18c0b191a2429bed namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: e38eded6-815d-425d-8610-d640ab8b266c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: ea78178f-4d40-476f-853d-8493e194cbd8 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-55ba8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:31Z" name: error-404-isvc-55ba8.18c0b191a242f9e3 namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: 7de2b8db-e03e-4d74-a36f-682f44aa22d4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: ea78178f-4d40-476f-853d-8493e194cbd8 kind: Event lastTimestamp: "2026-07-09T18:12:29Z" message: InferenceService [error-404-isvc-55ba8] is Ready metadata: creationTimestamp: "2026-07-09T18:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:29Z" name: error-404-isvc-55ba8.18c0b19f212ded33 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 1a8dea63-2138-4cca-89af-0177d32b713e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:59:56.174850Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0efbe4943a7 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 2e42a7ba-1bf1-4b8d-9ae5-5137a852e9e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5bd2e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0efc5f18183 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 96936499-f988-4702-afd6-f1b0107ea182 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0effc4472fb namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: 9c8ea691-9712-497d-8b66-2b179924604e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0effdaa2cc8 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 18cd9ecb-bd0f-44f9-9a81-e9d616537486 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0f0010f6399 namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: f8b78778-7879-4fb8-8e82-979c69305efd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0f00183f0ce namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 25e61c2f-18aa-4ec4-a7d8-761fc4d2f54c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0f0018db725 namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: cecdba2a-78dd-42b7-be3f-29fa87832f3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0f00463d353 namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 82485bd3-7a7c-4b87-997b-61e3c91e4bbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T17:59:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:57Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0f004d24ddb namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: 9aed958c-1c37-4751-a8c6-f2afd0fdad50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T18:00:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:34Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b0f032eea65e namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 8ac1a3b7-0dbe-4882-be07-7a06394230cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b10017aedabe namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 419581dc-1d47-4147-be8e-ac12406e8c81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b10017af641e namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: a147c579-e092-4be8-b909-6bb641e486d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 4277b038-4f02-471b-8133-e36bf4336e49 kind: Event lastTimestamp: "2026-07-09T18:01:09Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:09Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn.18c0b100bb1d6129 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 4c4307c5-1e5b-45e3-89ee-274c29f63f9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5bd2e-predictor-659bf7cb45 namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: 3fc293a2-a314-4746-94c4-1eafb9290c89 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'Created pod: error-404-isvc-5bd2e-predictor-659bf7cb45-kwlgn' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-5bd2e-predictor-659bf7cb45.18c0b0efbd9f75dc namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: 2ef2faad-b175-430a-909f-0c44cece691c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19684" uid: 0d11e5e3-f3c9-4155-8901-a0fb47e6f8f7 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Scaled up replica set error-404-isvc-5bd2e-predictor-659bf7cb45 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-5bd2e-predictor.18c0b0efbb82bc13 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 552e4f41-c857-4524-95d6-47b72abea082 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: bf9f01aa-d15e-4006-b90c-a1ad2c1cb67e kind: Event lastTimestamp: "2026-07-09T18:00:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:11Z" name: error-404-isvc-5bd2e-predictor.18c0b0f33a80dda2 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: e3a0e99b-7d91-437a-82a7-4aaea89268fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: bf9f01aa-d15e-4006-b90c-a1ad2c1cb67e kind: Event lastTimestamp: "2026-07-09T18:00:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:11Z" name: error-404-isvc-5bd2e-predictor.18c0b0f33b4b328f namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: f7cf7363-af57-4c10-82e5-d5d5a01ea6ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: bf9f01aa-d15e-4006-b90c-a1ad2c1cb67e kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-5bd2e-predictor.18c0b0f6ba005836 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 412e1353-92ee-4bbc-bdc0-29e707cdb1fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: bf9f01aa-d15e-4006-b90c-a1ad2c1cb67e kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-5bd2e-predictor.18c0b0f6ba53b2fc namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: b4b69074-82f1-465b-91a9-03c59a1a60c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: d3b9c9a5-3d7c-4c6a-9f9b-1c8dcd141038 kind: Event lastTimestamp: "2026-07-09T18:00:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:44Z" name: error-404-isvc-5bd2e.18c0b0efbc01835d namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: fd32a685-c4fc-4055-9ca5-d729a939baf5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: d3b9c9a5-3d7c-4c6a-9f9b-1c8dcd141038 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-5bd2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5bd2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-5bd2e.18c0b0efbc66dd0c namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 998cf18b-6b8a-4785-8cbb-40113d7a8179 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: d3b9c9a5-3d7c-4c6a-9f9b-1c8dcd141038 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5bd2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-5bd2e.18c0b0efbc67347e namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: f6337c0f-cd72-4a67-bdcb-36715f558cbf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: d3b9c9a5-3d7c-4c6a-9f9b-1c8dcd141038 kind: Event lastTimestamp: "2026-07-09T18:00:44Z" message: InferenceService [error-404-isvc-5bd2e] is Ready metadata: creationTimestamp: "2026-07-09T18:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:44Z" name: error-404-isvc-5bd2e.18c0b0faef1cdbbd namespace: kserve-ci-e2e-test resourceVersion: "20153" uid: 420cab14-17c4-4d51-945d-4dedc9985c7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:11:21.418737Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5d567-predictor-549f4c7f9-lbngd to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f4a011932 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: ca0fed0a-c955-41cb-9d70-d292c859d6f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25668" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f649082d1 namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: cb732d8f-83a1-4dec-b68a-f66d646adc0c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f66348a65 namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: 3d740893-c6f8-4a8c-9f34-0eaf974b829f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f688cb96e namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 63989790-81a0-4240-a765-fed20bd8a871 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f68faf0a8 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: e541737b-d8ad-4e0d-a9e6-c87ac7f820a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f690423fd namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: c3e17f8e-4dae-4ebb-87e3-c02319687672 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f6bd4e565 namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: 9a3b2bcd-bb7c-43e7-b94b-ed5aeeb3e2e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18f6c5fee1c namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 2c4fc5d8-a50c-4d9d-a349-96003ab5b269 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T18:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:12:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:09Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b18fb999373b namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 2193f112-c800-44d9-916a-b46df37140cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:28:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:28:50Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b28396d6822b namespace: kserve-ci-e2e-test resourceVersion: "33714" uid: 6e13d9bf-3861-4f46-aca4-37c53dac5060 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: be7ec986-945e-43cf-9881-67480013cb2c kind: Event lastTimestamp: "2026-07-09T18:28:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:28:50Z" name: error-404-isvc-5d567-predictor-549f4c7f9-lbngd.18c0b28396d70155 namespace: kserve-ci-e2e-test resourceVersion: "33715" uid: 2f028744-ca79-48bf-a4b2-cebd55cf1bc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5d567-predictor-549f4c7f9 namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: b7f8c2b3-7dcc-425c-be29-aafec28919a4 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created pod: error-404-isvc-5d567-predictor-549f4c7f9-lbngd' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor-549f4c7f9.18c0b18f49826fc9 namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 8da2e476-5818-4afc-a3fb-d34d7a61a847 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 0d7d6b95-2e66-40d3-8130-7d2b1aabe742 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Scaled up replica set error-404-isvc-5d567-predictor-549f4c7f9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567-predictor.18c0b18f47ed81f4 namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 2302b3c0-48c7-48f0-815a-321ddd8117db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25646" uid: e941a0d6-aba7-4512-b82e-5bd90cdb438a kind: Event lastTimestamp: "2026-07-09T18:11:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:51Z" name: error-404-isvc-5d567-predictor.18c0b192c6641cbf namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: bbed3a68-a2ce-4a46-a01b-3627caa178af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25646" uid: e941a0d6-aba7-4512-b82e-5bd90cdb438a kind: Event lastTimestamp: "2026-07-09T18:11:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:51Z" name: error-404-isvc-5d567-predictor.18c0b192c6c51691 namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 2f150b62-07c1-4b05-9b64-5e6c71505bb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: e941a0d6-aba7-4512-b82e-5bd90cdb438a kind: Event lastTimestamp: "2026-07-09T18:13:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:13:06Z" name: error-404-isvc-5d567-predictor.18c0b199c416e693 namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: 686b3bf1-afed-409b-beca-3ab78d4a8dc0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: e941a0d6-aba7-4512-b82e-5bd90cdb438a kind: Event lastTimestamp: "2026-07-09T18:13:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:13:06Z" name: error-404-isvc-5d567-predictor.18c0b199c47ac5b4 namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: a4fb71f4-35b7-4f5e-ba6d-f63690bd8f82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d567 namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: 3c3de319-662b-4ff7-9775-7e59c9992eb3 kind: Event lastTimestamp: "2026-07-09T18:12:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:19Z" name: error-404-isvc-5d567.18c0b18f47f61bd3 namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: b445b3e8-7797-49f1-b519-ab3bf291e2ca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d567 namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: 3c3de319-662b-4ff7-9775-7e59c9992eb3 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-5d567": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5d567": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567.18c0b18f4870a6ab namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 44732d7d-0689-4bcd-ae7a-bec3758e2e2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d567 namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: 3c3de319-662b-4ff7-9775-7e59c9992eb3 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5d567": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-5d567.18c0b18f4870ff39 namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: 9bea78b7-b30c-46da-90c9-efbd0aad22c8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5d567 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 3c3de319-662b-4ff7-9775-7e59c9992eb3 kind: Event lastTimestamp: "2026-07-09T18:12:19Z" message: InferenceService [error-404-isvc-5d567] is Ready metadata: creationTimestamp: "2026-07-09T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:19Z" name: error-404-isvc-5d567.18c0b19cc95a3895 namespace: kserve-ci-e2e-test resourceVersion: "26652" uid: 3498ce7d-bf20-4e0a-8d9a-6a2a56f2e82c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:49:11.616863Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-654f2-predictor-75c6649bd7-rkpbb to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:49:11Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b059aba2f00f namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 87753806-73ca-466d-9b5e-ab3ef8bf3d80 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:12Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:49:12Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b059c5a9f426 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 91568fbc-c986-41df-b0d6-1ca3283dece6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:12Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" metadata: creationTimestamp: "2026-07-09T17:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:12Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b059c7d0716a namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: 6a521596-69bc-4e81-8dc1-1119c35b2f26 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" in 15.72s (15.72s including waiting). Image size: 1334626400 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05d70d0a7e8 namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 2dff88b8-7bd3-4dcd-b17f-ce2ab0a88b9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05d74775f43 namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: d580ce77-41c5-4cb5-b935-1af5c58230f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05d74eb8d50 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 954637a5-9616-4816-939f-dda64886d32d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05d74f8414f namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: ccd49ec4-f97a-4266-a09a-601e50d73b78 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.553s (2.553s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:30Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05e0d26be35 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: e866669e-95ba-46e4-8840-b253d15bf021 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:30Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05e107d46fa namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: d49a901c-7b02-40f3-84d6-5849eb6b38d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:49:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:30Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05e10f49229 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 818d32f8-1817-4a3f-b661-b4d7852a4716 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:50:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:07Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b05e43bf4675 namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 40b888a5-d5e9-4ad5-8468-03bcb3a53095 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b06e8395fbdc namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: 64726cfe-1084-4230-9eec-e31aef3e4ae5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b06e83966a38 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 2373afef-b473-40e8-9926-f9649c5b2e2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 97e1342b-9aef-469a-b803-1711f8549184 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb.18c0b06ecc0699a5 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 587d7a97-f920-4b43-b85e-8ec247ae5f4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-654f2-predictor-75c6649bd7 namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: c26f3698-3e33-440a-9174-a5ac7433343a kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: 'Created pod: error-404-isvc-654f2-predictor-75c6649bd7-rkpbb' metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: error-404-isvc-654f2-predictor-75c6649bd7.18c0b059ab124912 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 2c471b4f-07f2-4efb-8d80-8cf1dc077b1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 5e18958b-10e3-4f42-a0ab-f3768890d32c kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Scaled up replica set error-404-isvc-654f2-predictor-75c6649bd7 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: error-404-isvc-654f2-predictor.18c0b059a72a07e2 namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 4aa713f8-03dc-4d87-bb03-9d3ee11c4c5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 7e4d840d-8946-42cd-87f7-0ad95ab0d3f5 kind: Event lastTimestamp: "2026-07-09T17:49:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:56Z" name: error-404-isvc-654f2-predictor.18c0b05d25a0e238 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 72c2649f-8d32-4ddb-a2c9-9ba6cdee19ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 7e4d840d-8946-42cd-87f7-0ad95ab0d3f5 kind: Event lastTimestamp: "2026-07-09T17:49:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:56Z" name: error-404-isvc-654f2-predictor.18c0b05d2615f8ca namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 31370686-e8ec-4af7-a6db-e6ee33114fb8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 7e4d840d-8946-42cd-87f7-0ad95ab0d3f5 kind: Event lastTimestamp: "2026-07-09T17:50:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:26Z" name: error-404-isvc-654f2-predictor.18c0b067a36759c7 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 907a3d6f-4075-449d-94c1-c9ec9eea31ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 7e4d840d-8946-42cd-87f7-0ad95ab0d3f5 kind: Event lastTimestamp: "2026-07-09T17:50:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:26Z" name: error-404-isvc-654f2-predictor.18c0b067a3d01267 namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: b3643b7e-178b-4582-985b-ab3a08dfe7fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-654f2 namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 5ebf8db4-1448-4409-b688-f85a6c0e1c6b kind: Event lastTimestamp: "2026-07-09T17:50:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:17Z" name: error-404-isvc-654f2.18c0b059a729a34b namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: ae40a76e-c193-4f5c-a3e6-57c16bbcc167 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-654f2 namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 5ebf8db4-1448-4409-b688-f85a6c0e1c6b kind: Event lastTimestamp: "2026-07-09T17:50:17Z" message: InferenceService [error-404-isvc-654f2] is Ready metadata: creationTimestamp: "2026-07-09T17:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:17Z" name: error-404-isvc-654f2.18c0b06903ce2cfc namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 59b5c3d2-8b50-4d2f-9acc-e5a54b016a63 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:02:06.805207Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e287678a3 namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 50f141fb-6416-4140-b149-e83e0d122c9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-67922-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e33ca3eda namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 836d733a-4326-4bb4-85dd-6aca386b016e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e6645be6e namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: c2fa2efc-da6e-4b8c-beac-866acfe772c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e67c14c07 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 6d54e90e-fb67-4aee-b98b-224abe2de601 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e6aa52043 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 4b68101e-4cf5-43c2-9c45-47b0c47c779f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e6b350149 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 71e18576-32f0-4519-bcae-d98491e701d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e6b409903 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 1c9109f2-a91a-40dd-84a3-f88a86da0dd7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e6dfce1e3 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 4ae76900-d469-413a-a81e-62e41ed8b55a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10e6e6a39ec namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 1bc87867-72ce-43bc-85d2-be5cba0098d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:02:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:45Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b10ed976db7b namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: 59cdff12-dc64-42c6-abd3-bbd23430e219 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b18f44302342 namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: b78a0b2d-2b7e-43d0-89b7-f5acbe877b44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 6a8237ad-4fb6-443d-a9a7-54f8141e16b3 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7.18c0b18f4430ad2d namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: ea5b89a5-3ba7-432b-aa3b-0a2ba3e2a005 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-67922-predictor-5b8d4977c7 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 9d3945b1-6053-4b42-aec2-ee297ccccf4d kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'Created pod: error-404-isvc-67922-predictor-5b8d4977c7-k7vt7' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-67922-predictor-5b8d4977c7.18c0b10e280b19eb namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 920cd628-26dd-4b7f-97ce-46ea7e4de917 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: fd975422-872c-4908-8549-05f9a3e4130a kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Scaled up replica set error-404-isvc-67922-predictor-5b8d4977c7 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-67922-predictor.18c0b10e262ecba8 namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 0b9ba531-a633-4ce6-846e-59ae6cb6bd5d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: e00f6b4b-0beb-4e17-bfac-1d9e55a39111 kind: Event lastTimestamp: "2026-07-09T18:02:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:36Z" name: error-404-isvc-67922-predictor.18c0b111a4feec57 namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 85a7f4f7-395a-49c9-bcb7-5f139126e507 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: e00f6b4b-0beb-4e17-bfac-1d9e55a39111 kind: Event lastTimestamp: "2026-07-09T18:02:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:36Z" name: error-404-isvc-67922-predictor.18c0b111a5636aa8 namespace: kserve-ci-e2e-test resourceVersion: "21645" uid: e87b151c-643d-4635-87dd-bda87530a362 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: e00f6b4b-0beb-4e17-bfac-1d9e55a39111 kind: Event lastTimestamp: "2026-07-09T18:03:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:21Z" name: error-404-isvc-67922-predictor.18c0b118a2c9a3b7 namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: c1d882b8-f84b-4a53-aeac-33d1b578be5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: e00f6b4b-0beb-4e17-bfac-1d9e55a39111 kind: Event lastTimestamp: "2026-07-09T18:03:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:21Z" name: error-404-isvc-67922-predictor.18c0b118a32ff42e namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 2351c154-2033-4581-b4ca-16651f75c813 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 485c733c-49f8-4f36-b037-442019240665 kind: Event lastTimestamp: "2026-07-09T18:02:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:55Z" name: error-404-isvc-67922.18c0b10e268968c7 namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: 29b02714-e9a0-4479-96f3-e5b6bcf00fbd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 485c733c-49f8-4f36-b037-442019240665 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-67922": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-67922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-67922.18c0b10e26f52a8c namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 2464babd-2d62-4945-99e8-c003902a1c19 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 485c733c-49f8-4f36-b037-442019240665 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-67922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:06Z" name: error-404-isvc-67922.18c0b10e26f581c9 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 5bdacffb-f2ac-4d81-8c13-acb2ab4cba16 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 485c733c-49f8-4f36-b037-442019240665 kind: Event lastTimestamp: "2026-07-09T18:02:55Z" message: InferenceService [error-404-isvc-67922] is Ready metadata: creationTimestamp: "2026-07-09T18:02:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:55Z" name: error-404-isvc-67922.18c0b11994fee8b8 namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: e8115089-371b-4fd0-a60b-335cba76b9fb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:51:41.542308Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:51:41Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07c93e387a7 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 16f70b99-2517-4005-941d-5bf9eb5cbebb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:51:41Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07cae2a2164 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 047c3e17-f337-4df5-985f-3afc36f6ca88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07caf8910ae namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 12a02462-6f36-47e8-8ff0-25e6efc591e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07cb25f67bf namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: eeb48147-c765-480a-a61d-44631a449d60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07cb2ccbb3c namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: 697654c3-bf34-48de-9084-f9f65f5ed5d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07cb2d65924 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 613c8d44-2144-47be-9f9e-644ec6c90c59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07cb5b0fdca namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: b9ac4585-7c85-4d0f-a4de-54aafa3304b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07cb6224c9d namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 7e4be954-7edc-4e34-a00e-6926907ecf2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T18:00:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:58Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b07ce5c13cb9 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: fb5d768c-b3ea-46af-9f01-4c92721b5bf3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b0fdbabb3f8b namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 7646e5c5-5b71-4090-b3e0-71c3e8074bdb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b0fdbabbb237 namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 406c424a-f69a-445f-9312-c73b84d67511 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 5c22b1bd-0f20-4bd8-8323-1467b767b87d kind: Event lastTimestamp: "2026-07-09T18:00:58Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:58Z" name: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb.18c0b0fe5a2cede1 namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: fc8a3b9a-75b6-426e-947b-f80a7d833172 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-71ac7-predictor-866d48d8f9 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 34ec5fb5-a3a9-4d78-b1ea-fd621859bdb6 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'Created pod: error-404-isvc-71ac7-predictor-866d48d8f9-r2xqb' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:41Z" name: error-404-isvc-71ac7-predictor-866d48d8f9.18c0b07c92f32a7a namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 32144c32-08d6-455c-87f9-88a7b9e34ea6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 1b66bb29-b75c-4f9b-9fe0-f4656ea61b90 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Scaled up replica set error-404-isvc-71ac7-predictor-866d48d8f9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:41Z" name: error-404-isvc-71ac7-predictor.18c0b07c8e653937 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 41ad9a12-fd2b-48f4-bcdb-a153a79f1e33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: f98a2833-b860-47e5-9772-2766e28448b8 kind: Event lastTimestamp: "2026-07-09T17:52:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:11Z" name: error-404-isvc-71ac7-predictor.18c0b0800c3d9f75 namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: 81754707-50d0-4725-9ded-12e2cd9612f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: f98a2833-b860-47e5-9772-2766e28448b8 kind: Event lastTimestamp: "2026-07-09T17:52:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:11Z" name: error-404-isvc-71ac7-predictor.18c0b0800ca8caa8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 3f828bf3-4e5f-408d-a954-4ce3f8342493 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: f98a2833-b860-47e5-9772-2766e28448b8 kind: Event lastTimestamp: "2026-07-09T17:52:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:56Z" name: error-404-isvc-71ac7-predictor.18c0b0870a5441ac namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: 6a0264cb-ffea-4049-a941-acc2be66f107 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:52:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: f98a2833-b860-47e5-9772-2766e28448b8 kind: Event lastTimestamp: "2026-07-09T17:52:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:56Z" name: error-404-isvc-71ac7-predictor.18c0b0870ac1903a namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: eeb26e2c-f471-431a-a622-26cd8aa40a89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: dd594e75-75cb-400d-8b42-a4fe0e4ef233 kind: Event lastTimestamp: "2026-07-09T17:52:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:29Z" name: error-404-isvc-71ac7.18c0b07c8e4efb78 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 28414eae-6338-4a55-9b3c-06ce1de292a8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: dd594e75-75cb-400d-8b42-a4fe0e4ef233 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-71ac7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-71ac7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:41Z" name: error-404-isvc-71ac7.18c0b07c8f23ecf1 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: fffac234-1860-4add-a7d0-87de4ff24a36 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: dd594e75-75cb-400d-8b42-a4fe0e4ef233 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-71ac7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:41Z" name: error-404-isvc-71ac7.18c0b07c8f246228 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 85acb40a-635d-4c3a-ae13-0460269d4a89 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: dd594e75-75cb-400d-8b42-a4fe0e4ef233 kind: Event lastTimestamp: "2026-07-09T17:52:29Z" message: InferenceService [error-404-isvc-71ac7] is Ready metadata: creationTimestamp: "2026-07-09T17:52:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:29Z" name: error-404-isvc-71ac7.18c0b087a15ffc31 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 0a6ba846-844a-4a99-8aad-c036ffe26378 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:50:41.429542Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-785cd-predictor-c6659f9bc-ptmcr to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06e94e38991 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: f0af5bb7-5911-4c17-9ac4-20964be52e80 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-785cd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06e9dc725fd namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 18d2b979-fd5d-4a06-b018-4bc23f2b1f85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06ed2c46967 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 017426c3-d89d-404b-94b0-8a7643bc00e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06ed42f287f namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f034a92b-7873-4a68-af19-1d9b3b31f72f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06ed719b328 namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 6c85169d-91f3-4ea2-9ed2-58ccf58e33ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06ed7863d19 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 5f896664-77f2-4b32-916d-0fff930cc09f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06ed790377f namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: a9080e37-9779-440c-bc96-0afd6863706c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06eda88aa01 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: ad9fc832-0147-48c3-a1ae-d9e45ff56705 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06edafb614a namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 356d937e-aa10-4ee7-8a53-e19e213186ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:59:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:58Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b06eddc0083e namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: fb3ae5dc-8b2c-4e13-84d3-dba052de3325 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b0efc1438782 namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 45b0e546-605f-44d3-8f07-a6cdc2d2b20b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b0efc143f584 namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: a45e0a14-c327-4689-baad-4fd458ea01f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: f243305b-4440-4c3d-bf34-ae1bcb2c2ec5 kind: Event lastTimestamp: "2026-07-09T17:59:58Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:58Z" name: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr.18c0b0f052295cdf namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: f8de6d50-211d-4541-8530-50c3dbd8b0b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-785cd-predictor-c6659f9bc namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: 5d7fd518-3d3b-4355-91d8-4f2ad3d912ca kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'Created pod: error-404-isvc-785cd-predictor-c6659f9bc-ptmcr' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-785cd-predictor-c6659f9bc.18c0b06e945c4a6b namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 90c6532c-37db-432e-a161-d434f1349a81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 2f83f5fd-a4c6-4dbd-a843-8e18cf8e0281 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Scaled up replica set error-404-isvc-785cd-predictor-c6659f9bc from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-785cd-predictor.18c0b06e927f12de namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: 20be1ad4-e720-4cab-b965-8690b7169810 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:50:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: fe20a301-fb23-4d40-9ad7-38b49ebf2891 kind: Event lastTimestamp: "2026-07-09T17:51:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:11Z" name: error-404-isvc-785cd-predictor.18c0b07211e1d944 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 9950ec2b-955a-4735-8f7a-2aea2c05fafd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:50:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: fe20a301-fb23-4d40-9ad7-38b49ebf2891 kind: Event lastTimestamp: "2026-07-09T17:51:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:11Z" name: error-404-isvc-785cd-predictor.18c0b07213054769 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 465939ee-40e0-4f33-85a9-0eb419e11e5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:51:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: fe20a301-fb23-4d40-9ad7-38b49ebf2891 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: error-404-isvc-785cd-predictor.18c0b0791087eb99 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: f5360637-2b8a-4e02-ad61-d57e5aa99ba1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:51:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: fe20a301-fb23-4d40-9ad7-38b49ebf2891 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: error-404-isvc-785cd-predictor.18c0b07910f657a0 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 2af17398-bf97-41e7-b5f3-ee0ce67456de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: e9651be7-28f2-4baa-9e79-ea143347ab78 kind: Event lastTimestamp: "2026-07-09T17:51:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:28Z" name: error-404-isvc-785cd.18c0b06e928a4202 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 813d2893-b3d4-4335-981a-4482fbd0ee88 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: e9651be7-28f2-4baa-9e79-ea143347ab78 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-785cd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-785cd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-785cd.18c0b06e93248dc3 namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: fd302e98-a1d5-449c-9e68-97309b9dc960 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: e9651be7-28f2-4baa-9e79-ea143347ab78 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-785cd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:41Z" name: error-404-isvc-785cd.18c0b06e9324e45f namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: ac10bc3c-e41a-4346-bbaa-5da75e0cc0ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: e9651be7-28f2-4baa-9e79-ea143347ab78 kind: Event lastTimestamp: "2026-07-09T17:51:28Z" message: InferenceService [error-404-isvc-785cd] is Ready metadata: creationTimestamp: "2026-07-09T17:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:28Z" name: error-404-isvc-785cd.18c0b0799a1ea10c namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 48e16bd6-0d73-4576-abd9-ba09823f41d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:49:11.231056Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14219" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf to ip-10-0-133-63.ec2.internal metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b05994a3dbfc namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: a0c16413-2305-41c5-b5e6-c1f6cc3c920e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b059adf6c31a namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: a7f23886-5176-4603-a95a-45dadd75d26a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b059b038135e namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 85375bac-4aba-456a-bf44-e178c6639ee9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b059b3143222 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: b714f341-357f-49c5-ba7b-e0c3b90b8a59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b059b384997f namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 632d9c8f-5c48-47fd-b8ec-94ff43aa8549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:15Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" metadata: creationTimestamp: "2026-07-09T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:15Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b05aa792490b namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: b77a8309-02b1-4057-bbcb-e779ccfa9644 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:42Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" in 26.827s (26.827s including waiting). Image size: 1555846059 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:42Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b060e69efc81 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: db161906-4e83-4ba3-8ad2-fe70ebe589f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:49:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:42Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b060ee23edae namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: 471eb156-0c7a-4b95-8dfb-c309cabf9d73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:49:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:42Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b060ef90c644 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: a547ceba-aed4-49bf-92f0-36ab3bb23665 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T17:49:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:42Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b060efa4db68 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 4ecaad84-5325-489c-aa46-16510a1cb210 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.366s (1.366s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b061411976e7 namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 6cd6c881-d211-4608-9616-1033d0c9248e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b0614462d34d namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: c00ffc99-6221-4155-8e33-347c16698d6a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b06144e51600 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: aade261f-10df-428c-93eb-cb38407c3aa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:50:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:51Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b06173df267c namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: adff3a3a-3fa0-4573-bfbf-b8771d35e9a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b07c84617e8f namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: b422d35f-c0ac-4ae9-bbbc-2bdb000d9f97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b07c8462b638 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 7ecdbbb8-d54c-4600-b81b-eeadb968a78c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: eab88989-e0f2-4690-857a-e9bb6e6d93c5 kind: Event lastTimestamp: "2026-07-09T17:51:46Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:46Z" name: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf.18c0b07da08080fa namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 8d763c1d-897a-439b-82cd-5d2852e64a59 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-64f67649c4 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 751b8242-452e-422b-ba04-d59ba39a1673 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-64f67649c4-7k6gf' metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor-64f67649c4.18c0b05993b750a9 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 9d278995-f78f-46d8-9f0f-92fa2cd36eb5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 1adb0e13-521a-4485-9a39-4b4b3b5afe30 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-64f67649c4 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-1-predictor.18c0b0598d47f72a namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: abce3cb1-8a8d-4667-b315-1c0783d7f109 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 71bcc9c4-e1a0-4d43-8f58-ac7f3799ddbc kind: Event lastTimestamp: "2026-07-09T17:50:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:26Z" name: isvc-sklearn-graph-1-predictor.18c0b05d0bd2c7e7 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 78b6cdf5-3843-465a-a920-30328b93aa79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 71bcc9c4-e1a0-4d43-8f58-ac7f3799ddbc kind: Event lastTimestamp: "2026-07-09T17:50:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:26Z" name: isvc-sklearn-graph-1-predictor.18c0b05d0c46202d namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 5975f0e3-6546-4596-90d1-8232118ccf64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 71bcc9c4-e1a0-4d43-8f58-ac7f3799ddbc kind: Event lastTimestamp: "2026-07-09T17:51:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:26Z" name: isvc-sklearn-graph-1-predictor.18c0b06e864643bc namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: d99b4ebd-d899-498f-89ba-a48e08d4ac98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 71bcc9c4-e1a0-4d43-8f58-ac7f3799ddbc kind: Event lastTimestamp: "2026-07-09T17:51:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:26Z" name: isvc-sklearn-graph-1-predictor.18c0b06e86daf32e namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: a5e889b6-9a50-4920-8e7e-b998b20b6a86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 95a8cae6-09fd-4112-887a-acd178d8e42d kind: Event lastTimestamp: "2026-07-09T17:51:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:01Z" name: isvc-sklearn-graph-1.18c0b0598d54da03 namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 3993d55c-99e6-4dc1-92e8-af96e8b97d12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: 95a8cae6-09fd-4112-887a-acd178d8e42d kind: Event lastTimestamp: "2026-07-09T17:51:01Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-09T17:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:01Z" name: isvc-sklearn-graph-1.18c0b0732cb3a272 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: 1ac4ebba-d319-4113-9d6c-1f2f6792eb84 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:49:11.822690Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 to ip-10-0-133-63.ec2.internal metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b059b7e790b1 namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: ce686ad5-23f3-4f6a-a7ca-fcb504d435a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:12Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:49:12Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b059d1bb7850 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: 1ebe3afb-adc1-41c4-b344-54fe3e6b7097 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:12Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b059d354daa5 namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 1f0dd28e-c410-4357-8c00-f11a0a39b0c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:12Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b059d6575bdb namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: 5d57bf6b-a3d6-4678-af67-4ce06b70cc59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:12Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b059d6cd2ed3 namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: cea113d7-c9b0-422f-b6e9-2b920030ca12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:16Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" metadata: creationTimestamp: "2026-07-09T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:16Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b05ae42684dc namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 2492e02c-902d-4063-be45-8de1893644d1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" in 24.423s (24.423s including waiting). Image size: 1555846059 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b06093e9d71f namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 75494d5f-3f52-42ee-8dc6-565e579ac862 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b06098734ccb namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: dae389f6-447d-40fa-80e9-13be0d2d627b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b06099a2093f namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ec4c6f37-00a4-461c-8384-77d4e3f1ca34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b06099ae3779 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 3af27980-3dcc-4f23-b921-50ccf288b7b6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.79s (2.79s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b061400192de namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 899f5293-9891-4a57-b27b-2b90e0818503 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b0614432647d namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: d59a14eb-1581-4a4f-ba38-4d6252d08994 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b06144c06bcc namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 77a5c281-1914-4e6c-8c6c-c6be6f0bd62f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:50:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:51Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b06173cbe104 namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 0e54c088-075a-41c3-88e8-770fc8a3c652 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b07c7ca7bdad namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: f7f58c77-95ad-4a6c-beb5-387a3b2658a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b07c7ca835ec namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 1bd4388f-9028-4130-8f6a-af8817684a2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 01e3411a-227b-4060-9de4-184750ea161d kind: Event lastTimestamp: "2026-07-09T17:51:47Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-09T17:51:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:47Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27.18c0b07ddc202d7d namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 3fbf17e1-7497-403c-a89a-12211c04f956 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6d7745cf6b namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: 8af347c6-8d65-4f1c-ae06-70b2baf21561 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6d7745cf6b-7hk27' metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-2-predictor-6d7745cf6b.18c0b059b726824e namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 784337d4-2bad-4346-a854-d004c7d3c575 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 8a45f3c2-d594-4c8c-b5de-b65b1f3ce497 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6d7745cf6b from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: isvc-sklearn-graph-2-predictor.18c0b059b1ae37d3 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 1e34f077-8a30-4ef5-86d6-bc78b429863b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 79482d9a-33e7-4133-aa96-4de99dacfaa6 kind: Event lastTimestamp: "2026-07-09T17:49:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:56Z" name: isvc-sklearn-graph-2-predictor.18c0b05d2fe3c63a namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 5a51991e-be79-4593-a73e-520cba32289c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 79482d9a-33e7-4133-aa96-4de99dacfaa6 kind: Event lastTimestamp: "2026-07-09T17:49:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:56Z" name: isvc-sklearn-graph-2-predictor.18c0b05d305e7dd8 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: d2ace172-d197-4226-aab0-cbf5294ccec8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 79482d9a-33e7-4133-aa96-4de99dacfaa6 kind: Event lastTimestamp: "2026-07-09T17:51:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:26Z" name: isvc-sklearn-graph-2-predictor.18c0b067ac9b7191 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 740c0fdc-4154-45ff-a774-2e98f79565ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 79482d9a-33e7-4133-aa96-4de99dacfaa6 kind: Event lastTimestamp: "2026-07-09T17:51:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:26Z" name: isvc-sklearn-graph-2-predictor.18c0b067ad0588aa namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: c752ebbe-0bd6-453c-8c92-2d47973d55d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: 266999b6-c84b-4a1b-acc7-f20bc0ee6133 kind: Event lastTimestamp: "2026-07-09T17:51:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:01Z" name: isvc-sklearn-graph-2.18c0b059b19e576a namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: c52ac4c6-ea71-4b12-9b85-c69b06143bb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 266999b6-c84b-4a1b-acc7-f20bc0ee6133 kind: Event lastTimestamp: "2026-07-09T17:51:01Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-09T17:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:01Z" name: isvc-sklearn-graph-2.18c0b073309d1cbd namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 9c07c502-5929-446a-badf-41faab492ade reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:49:11.425736Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-2nzjf to ip-10-0-133-63.ec2.internal metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b059a03e7416 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 7bb888da-7826-40c9-9391-7ef4dd2943a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b059ba1fec3b namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: ab09ba53-f794-46d2-b41f-504d71523338 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b059bc40ced1 namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: e1add59f-72a9-4289-bd34-aa1958271664 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b059bed82656 namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: b820b670-280e-4c6b-aee3-1e855ee7d186 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b059bf483a9d namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: b58c9a68-8466-4a4d-8737-ae5721407770 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:15Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-09T17:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b05aa7ac2f7b namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: 2df78443-41de-40da-b14d-a53ac3044e1a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 25.789s (25.789s including waiting). Image size: 1305579259 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b060a8dcd0a8 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 57ce4fc8-2ed5-4779-8c75-692b47e5da54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b060af873d39 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 3b9752fa-15be-43a2-ac6d-68c3758ae2d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b060b0d44598 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 4adbfb39-d758-49c4-9ada-ef92f3d53ada reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T17:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b060b0e2ae1e namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 6c2bff59-9d2d-40e3-9815-984877373ed1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.378s (2.378s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b0613ea67068 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 281161de-992a-4301-9a39-3d23c08182ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b06144030ea8 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 29a6e6d2-ef9b-499f-b632-c2966b691149 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:49:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b061448dfeb1 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: c97c1eb1-186c-41b8-81ff-aafba6ddb4ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-09T17:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b06173fedc9e namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 0e0f4355-e8ea-461c-bd45-3d8216e2aa66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b07c7948a2e3 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: d9ff0ea2-c2e6-435d-80c4-407b5b0bb987 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 423e843b-dfbe-4457-9ed1-6d9e3deec950 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2nzjf.18c0b07c79491cbb namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 4e4180f4-edb4-4bb7-8fff-43ea69e3fff5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-63.ec2.internal source: component: kubelet host: ip-10-0-133-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 9395eca8-24bc-427e-8a7d-a101613e500c kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-2nzjf' metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c0b0599f736e67 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: eb3682c8-f039-44c0-967e-0260c043453f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 3e7da885-bacc-4b5c-b335-4960b75b5f49 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:11Z" name: isvc-xgboost-graph-predictor.18c0b05999e21064 namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 879d1ad6-802d-4efe-bdee-73d2544d8eb9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: 058d59d6-8f19-4cc4-8628-9ce063e4d008 kind: Event lastTimestamp: "2026-07-09T17:49:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:56Z" name: isvc-xgboost-graph-predictor.18c0b05d190b83c1 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: f3dd20b9-6705-49b6-97cd-25c0cc981678 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: 058d59d6-8f19-4cc4-8628-9ce063e4d008 kind: Event lastTimestamp: "2026-07-09T17:49:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:56Z" name: isvc-xgboost-graph-predictor.18c0b05d19831308 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: cd90ac61-b032-4e02-a954-b6d51ae514b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 058d59d6-8f19-4cc4-8628-9ce063e4d008 kind: Event lastTimestamp: "2026-07-09T17:51:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:11Z" name: isvc-xgboost-graph-predictor.18c0b06795c6e533 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 167dd861-fca7-422b-b892-b0837960fa4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 058d59d6-8f19-4cc4-8628-9ce063e4d008 kind: Event lastTimestamp: "2026-07-09T17:51:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:11Z" name: isvc-xgboost-graph-predictor.18c0b06796355cfc namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 7f3f5269-53b2-4ae1-9d4f-ac75676ad0ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: d0028de1-2dc5-4dd5-a55d-024c10a4b4db kind: Event lastTimestamp: "2026-07-09T17:50:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:51Z" name: isvc-xgboost-graph.18c0b0599a9344a2 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 8c6dc955-5404-41e0-9ade-c591e297e5ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: d0028de1-2dc5-4dd5-a55d-024c10a4b4db kind: Event lastTimestamp: "2026-07-09T17:50:51Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-09T17:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:51Z" name: isvc-xgboost-graph.18c0b070d9435663 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 879f7fee-0cf5-4c98-ab4f-e47862bc8130 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:51:30.913432Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-659948ff55-77x5x to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T17:51:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:51:30Z" name: model-chainer-659948ff55-77x5x.18c0b07a1a5bddc2 namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 9151e82c-4492-4d74-a846-6f1a136d6be4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:51:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:51:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:31Z" name: model-chainer-659948ff55-77x5x.18c0b07a25c0b5a3 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 9ce27b36-301b-4259-ae4c-5215632027e3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:51:31Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:51:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:51:31Z" name: model-chainer-659948ff55-77x5x.18c0b07a5811c613 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: dd564a8a-db9a-46cd-9c30-86eb9687b574 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:51:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:31Z" name: model-chainer-659948ff55-77x5x.18c0b07a5967b703 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: efedfe12-757d-471e-a964-b00d0442dbb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:51:32Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-09T17:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:32Z" name: model-chainer-659948ff55-77x5x.18c0b07a5d377b4d namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 328af361-3907-4df4-bfc5-9e0b57b881df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:51:32Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-09T17:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:32Z" name: model-chainer-659948ff55-77x5x.18c0b07a5db2d69b namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 71edefd6-4a20-48f7-8b19-20f3fc0369b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: model-chainer-659948ff55-77x5x.18c0b07c73e59390 namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: f5f2f490-761a-4de9-b0d1-01316bf591c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659948ff55-77x5x namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: c0ec6369-4d1d-4d65-b38f-dec00dd62844 kind: Event lastTimestamp: "2026-07-09T17:52:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:08Z" name: model-chainer-659948ff55-77x5x.18c0b07d0973ae55 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 8977b8a5-a69b-4927-bbd1-efd8804172c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-659948ff55 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 8f9c380d-db65-48c6-95b7-705beb84e293 kind: Event lastTimestamp: "2026-07-09T17:51:30Z" message: 'Created pod: model-chainer-659948ff55-77x5x' metadata: creationTimestamp: "2026-07-09T17:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:30Z" name: model-chainer-659948ff55.18c0b07a19ad828e namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 90d31fa3-ef5d-49a3-b392-0dacfa996929 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 56a7d660-8d28-4900-8ac2-19eb78a2953f kind: Event lastTimestamp: "2026-07-09T17:51:30Z" message: Scaled up replica set model-chainer-659948ff55 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:30Z" name: model-chainer.18c0b07a1864901e namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 21dee7ce-191c-4f6f-9871-7df2c6db6580 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 52976450-6a68-40fd-8c3b-6c81eacf8385 kind: Event lastTimestamp: "2026-07-09T17:51:31Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-09T17:51:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:31Z" name: model-chainer.18c0b07a1febfc26 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: c00f8d44-ec3e-4e45-a567-a504a967235e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:01:56.171108Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-53a2e-5dd9575f9b-gddb4 to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:01:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:01:56Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10bae9ecdd9 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: e4a30f0a-c731-406a-869f-b065eb1ab1cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-53a2e-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:56Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10bb8537956 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: c8b95a30-4f89-4069-826f-bb2d4e4c670c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:01:57Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10bec270dbe namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: a0be46b0-5a48-46a7-af8d-cffa5553233d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a2e} kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:57Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10bedbece05 namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 1856a57d-010e-41ab-9f2c-f4127e8cefa6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a2e} kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: 'Created container: sequence-graph-53a2e' metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:57Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10bf0aacab2 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 398ea11e-eb9c-4928-b012-200fa52678ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a2e} kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: Started container sequence-graph-53a2e metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:57Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10bf11f7e28 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 7cc3dc4a-169b-4b0a-bbba-a6a52a34b356 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a2e} kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Stopping container sequence-graph-53a2e metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10e07eab8ed namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 8fcd5df3-2abf-4fa9-9c52-9e7456bd435f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a2e} kind: Pod name: sequence-graph-53a2e-5dd9575f9b-gddb4 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: 98b9d6c5-d6e7-4d52-b3e2-4a01c9f005fe kind: Event lastTimestamp: "2026-07-09T18:02:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:34Z" name: sequence-graph-53a2e-5dd9575f9b-gddb4.18c0b10eaf026f14 namespace: kserve-ci-e2e-test resourceVersion: "21617" uid: af2259d0-44dd-48f5-9da4-10ed07131403 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-53a2e-5dd9575f9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 05695c61-3272-442b-b1a1-bb7b3debc823 kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: 'Created pod: sequence-graph-53a2e-5dd9575f9b-gddb4' metadata: creationTimestamp: "2026-07-09T18:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:56Z" name: sequence-graph-53a2e-5dd9575f9b.18c0b10bae02bfa7 namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 545859ca-adf1-468d-b506-2b6c6f650613 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-53a2e namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: cc3f6a8f-85d9-4176-9b86-e74aba237756 kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: Scaled up replica set sequence-graph-53a2e-5dd9575f9b from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:56Z" name: sequence-graph-53a2e.18c0b10bacf6dff7 namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 61891d46-3163-40b5-a0d3-b3e5bb83b774 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-53a2e namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: edd18015-8c8a-4f46-a2ac-ed8a2b2a5617 kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: InferenceGraph [sequence-graph-53a2e] is Ready metadata: creationTimestamp: "2026-07-09T18:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:56Z" name: sequence-graph-53a2e.18c0b10bb1cbbdfd namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 29d48f42-2067-4ab4-b947-fa095c9626a0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:03:06.517453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-67922-65cfc7d557-46nnw to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:03:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:03:06Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b11c0f96dc44 namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: 233c6da5-c60a-4575-b4e1-560631becc5e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:03:06Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b11c2a01bb96 namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 41ee2f1e-9483-4551-a9e8-902b60e7a6c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67922} kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:06Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b11c2bea86db namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 02fb507f-5f41-42ca-ac67-f4921a9d25ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67922} kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: "2026-07-09T18:03:07Z" message: 'Created container: sequence-graph-67922' metadata: creationTimestamp: "2026-07-09T18:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:07Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b11c2f0a8c09 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 30b12313-9650-4ee5-841b-7b7f867420f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67922} kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: "2026-07-09T18:03:07Z" message: Started container sequence-graph-67922 metadata: creationTimestamp: "2026-07-09T18:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:07Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b11c2f7e81ae namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 0bdc4f9b-0724-4ab9-8f83-4d3432271ff3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67922} kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Stopping container sequence-graph-67922 metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b18f36f96467 namespace: kserve-ci-e2e-test resourceVersion: "25546" uid: 61e1ca46-6cfb-428f-9e9b-c58b5f258de1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67922} kind: Pod name: sequence-graph-67922-65cfc7d557-46nnw namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: f7d51d90-3774-4d90-8738-3c66b2a1a90b kind: Event lastTimestamp: "2026-07-09T18:11:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:48Z" name: sequence-graph-67922-65cfc7d557-46nnw.18c0b18fbcd3f192 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 07b2ba73-511d-4cf3-9727-89c30ce72ef3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-67922-65cfc7d557 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: ac6e41b6-714d-499f-8bca-32b92155d0f0 kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: 'Created pod: sequence-graph-67922-65cfc7d557-46nnw' metadata: creationTimestamp: "2026-07-09T18:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:06Z" name: sequence-graph-67922-65cfc7d557.18c0b11c0f0a9ea0 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: 85f230ef-f786-48b0-bbaa-4768831d5825 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-67922 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: b6a0f49c-2e4a-42a8-8fdf-7b4db940cafd kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: Scaled up replica set sequence-graph-67922-65cfc7d557 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:06Z" name: sequence-graph-67922.18c0b11c0e3373de namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 4f678c2f-e2e5-4a9e-8f28-f9affa9c3dae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-67922 namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 87cd5060-1756-4def-a684-5b640afddf8b kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: InferenceGraph [sequence-graph-67922] is Ready metadata: creationTimestamp: "2026-07-09T18:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:03:06Z" name: sequence-graph-67922.18c0b11c14038df4 namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 3179934f-510b-4621-a1bb-74c4b478461b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:03:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-67922 namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: b7ef0f56-296a-4197-8268-6add75a56927 kind: Event lastTimestamp: "2026-07-09T18:03:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:36Z" name: sequence-graph-67922.18c0b11f8c9e24dc namespace: kserve-ci-e2e-test resourceVersion: "22133" uid: 80db4f06-293c-44ba-8de2-1cdffcbaa5fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:03:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-67922 namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: b7ef0f56-296a-4197-8268-6add75a56927 kind: Event lastTimestamp: "2026-07-09T18:03:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:36Z" name: sequence-graph-67922.18c0b11f8cfb50a9 namespace: kserve-ci-e2e-test resourceVersion: "22134" uid: 9a51689d-83ce-4b25-8b38-654d3635bee2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:52:41.256798Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-71ac7-5968865f6c-x2nj8 to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b08a7b265b11 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 7c70e754-d324-4abb-af93-2b6d806fb04e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b08a9558f476 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 76205ced-c8f0-4763-912d-fc1787d132ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-71ac7} kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b08a96d315ea namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: e9bde201-265b-4ae2-8035-8fb4b300961b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-71ac7} kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: 'Created container: sequence-graph-71ac7' metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b08a99da2b00 namespace: kserve-ci-e2e-test resourceVersion: "16630" uid: 7b255bfe-99a2-4eb8-bb79-8bb1f46d7764 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-71ac7} kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: Started container sequence-graph-71ac7 metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b08a9a5cf40d namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 8f2dba5c-74f4-4d40-aa8c-e0b2ac08bed0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-71ac7} kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: "2026-07-09T18:00:55Z" message: Stopping container sequence-graph-71ac7 metadata: creationTimestamp: "2026-07-09T18:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:55Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b0fda9adf1fa namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: a50cd509-8e95-40b0-a7a8-578e1ae2191c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-71ac7} kind: Pod name: sequence-graph-71ac7-5968865f6c-x2nj8 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 13710333-bcd3-4c52-8faf-704c819476ff kind: Event lastTimestamp: "2026-07-09T18:01:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:22Z" name: sequence-graph-71ac7-5968865f6c-x2nj8.18c0b0fe16e9b084 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 66f2aef1-efe1-43fc-8a4b-1d6e0fd01e82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-71ac7-5968865f6c namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: c883e19f-9dac-4756-8357-e92a345b4777 kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: 'Created pod: sequence-graph-71ac7-5968865f6c-x2nj8' metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7-5968865f6c.18c0b08a7a61361e namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: 632093bb-9e65-4186-9f15-a6a0b31f4cb6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: 834a6c90-f92a-46ee-bc89-edb262c88a6d kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: Scaled up replica set sequence-graph-71ac7-5968865f6c from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7.18c0b08a793c2783 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: d1f7a08a-567e-4cdb-8dec-6b393e4f1ec6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 71dab934-ff99-4ec9-b703-28626ff32ee9 kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: InferenceGraph [sequence-graph-71ac7] is Ready metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7.18c0b08a80e3e4d8 namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 60787f82-77c9-4516-b5de-390cdbe7b322 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: 71dab934-ff99-4ec9-b703-28626ff32ee9 kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: 'Failed to update status for InferenceGraph "sequence-graph-71ac7": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-71ac7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7.18c0b08a8437b664 namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 2550b442-a850-462b-a9c8-ad5ffbe20a89 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: 71dab934-ff99-4ec9-b703-28626ff32ee9 kind: Event lastTimestamp: "2026-07-09T17:52:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-71ac7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:41Z" name: sequence-graph-71ac7.18c0b08a843814de namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: fca85592-49ef-44f8-8a91-effd9a8755ec reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 7cdf5aee-67ec-431d-89e3-f63dd6547170 kind: Event lastTimestamp: "2026-07-09T17:53:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:11Z" name: sequence-graph-71ac7.18c0b08df7b59e16 namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: 599f46b0-d3f6-426f-980f-1220fe9f1776 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 7cdf5aee-67ec-431d-89e3-f63dd6547170 kind: Event lastTimestamp: "2026-07-09T17:53:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:11Z" name: sequence-graph-71ac7.18c0b08df80da607 namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: c6d2a710-49ba-4ce1-96b9-cdae812c614d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:11:21.190787Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-07956-66ccc97694-fwn8j to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b18f3c6aded6 namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: 4dff7fff-e706-4c28-be4f-af981dd6a2a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25574" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b18f56830038 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: ecddf211-049a-45e4-99a9-a3dc9f892248 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-07956} kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b18f57d006d5 namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 4adedcd9-f7b3-4f7b-b032-4b890ef9762b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-07956} kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created container: splitter-graph-07956' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b18f5aa72085 namespace: kserve-ci-e2e-test resourceVersion: "25677" uid: 5fd4bfa4-d682-4704-801d-689d5daa0b00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-07956} kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Started container splitter-graph-07956 metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b18f5b2245e7 namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: 3a0f1ea8-d21b-47ca-b5a3-2b92c386a8a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-07956} kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Stopping container splitter-graph-07956 metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b191953ae097 namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: 567aa7b0-2b94-4027-8cca-b710147f665e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-07956} kind: Pod name: splitter-graph-07956-66ccc97694-fwn8j namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 9ebe67c6-85cc-45f1-a559-63329fdf7389 kind: Event lastTimestamp: "2026-07-09T18:11:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:58Z" name: splitter-graph-07956-66ccc97694-fwn8j.18c0b19212922602 namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: a2eefb90-14a8-4981-88d2-9efdd7060812 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-07956-66ccc97694 namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: d1662c84-4b62-4b4a-ba03-3033236ac050 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created pod: splitter-graph-07956-66ccc97694-fwn8j' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956-66ccc97694.18c0b18f3bbe1c57 namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: dba001f2-55cd-43ab-8618-723d024096dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-07956 namespace: kserve-ci-e2e-test resourceVersion: "25553" uid: 0545d9ad-e38c-4946-a632-6cf0634689b0 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Scaled up replica set splitter-graph-07956-66ccc97694 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956.18c0b18f3a790a92 namespace: kserve-ci-e2e-test resourceVersion: "25557" uid: 4b432f7b-3547-480b-8ede-f68e7f0a9297 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-07956 namespace: kserve-ci-e2e-test resourceVersion: "25587" uid: b4bb20b3-395a-4176-9b45-8ef05d89ceee kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: InferenceGraph [splitter-graph-07956] is Ready metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:21Z" name: splitter-graph-07956.18c0b18f3f7491b8 namespace: kserve-ci-e2e-test resourceVersion: "25593" uid: cbf2b6a5-fa29-4178-9859-ebad685e5fab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:12:31.486474Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-55ba8-6dcb4fc9cc-gvgrd to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:12:31Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b19f9a5df48a namespace: kserve-ci-e2e-test resourceVersion: "26798" uid: 07379788-cfa5-4c59-ba1a-8d90999f5b3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26802" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:12:31Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b19fb532ec3d namespace: kserve-ci-e2e-test resourceVersion: "26817" uid: cf99f118-1543-4e83-b3e9-b23cbd9c2bce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-55ba8} kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:31Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b19fb691b51e namespace: kserve-ci-e2e-test resourceVersion: "26819" uid: c3a53fd4-3d63-4905-9276-c59b62893a4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-55ba8} kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: "2026-07-09T18:12:32Z" message: 'Created container: splitter-graph-55ba8' metadata: creationTimestamp: "2026-07-09T18:12:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:32Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b19fba052c9a namespace: kserve-ci-e2e-test resourceVersion: "26820" uid: 06583109-c16e-4c9f-8427-307bac7fbba3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-55ba8} kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: "2026-07-09T18:12:32Z" message: Started container splitter-graph-55ba8 metadata: creationTimestamp: "2026-07-09T18:12:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:32Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b19fba854fcf namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 4f873000-85ca-4eac-bdeb-836e98c2615a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:20:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-55ba8} kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: "2026-07-09T18:20:45Z" message: Stopping container splitter-graph-55ba8 metadata: creationTimestamp: "2026-07-09T18:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:20:45Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b212b9aafc12 namespace: kserve-ci-e2e-test resourceVersion: "30316" uid: 1198ebd2-19f9-438f-b7d6-60cae154f84c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:20:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-55ba8} kind: Pod name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: b4515bb6-c307-4d3b-b5cf-51835d301110 kind: Event lastTimestamp: "2026-07-09T18:21:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:21:13Z" name: splitter-graph-55ba8-6dcb4fc9cc-gvgrd.18c0b2135c9f50ca namespace: kserve-ci-e2e-test resourceVersion: "30553" uid: d9ccde7a-38f7-4a93-a66e-b9273ef9b9f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-55ba8-6dcb4fc9cc namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 4bf795d3-0494-48c2-a32d-685cec2f4c06 kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: 'Created pod: splitter-graph-55ba8-6dcb4fc9cc-gvgrd' metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:31Z" name: splitter-graph-55ba8-6dcb4fc9cc.18c0b19f99ff4a4b namespace: kserve-ci-e2e-test resourceVersion: "26794" uid: 69138988-bba6-404b-b3a0-f2ce4a700949 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: 30de1993-ec5f-4211-a124-8ce7cae57628 kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: Scaled up replica set splitter-graph-55ba8-6dcb4fc9cc from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:31Z" name: splitter-graph-55ba8.18c0b19f996358eb namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: ad133704-6a66-44c3-bbbc-dbc0b0f64b7f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "26807" uid: f7ded7d4-caa5-41f0-ac19-142439be40fd kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: InferenceGraph [splitter-graph-55ba8] is Ready metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:31Z" name: splitter-graph-55ba8.18c0b19f9eef28af namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: a1805de9-555e-4437-8823-c2db83e7d2fa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 3708ee15-638a-4c21-959c-d7c923ddcb8a kind: Event lastTimestamp: "2026-07-09T18:12:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:46Z" name: splitter-graph-55ba8.18c0b1a317de4e69 namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 10df2dc9-8925-4246-8e98-7e287482e015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 3708ee15-638a-4c21-959c-d7c923ddcb8a kind: Event lastTimestamp: "2026-07-09T18:12:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:46Z" name: splitter-graph-55ba8.18c0b1a31840c298 namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: 020141bd-a1bd-484a-a75e-2557e7e9ba4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:10:21.203827Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24997" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-07956-predictor-649f94cfd6-mgsrn to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b18144ea91d8 namespace: kserve-ci-e2e-test resourceVersion: "25004" uid: 5d20ee14-72ac-4530-aa58-e967b5a3465e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b1815ee51e13 namespace: kserve-ci-e2e-test resourceVersion: "25039" uid: 05d6b4af-1513-40c1-8916-ba5c4713ade4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b18160601753 namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: 37e99dae-de30-4927-93d6-abac174bf0f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b1816336945b namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 4feaa4f5-160e-4183-80f4-187dbc9cda9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b18163ada32d namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: ac51c0e8-7758-4fb8-9382-9cf2844884ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b18163b9922e namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: f96a6ff0-be08-4b59-a157-902ae717e975 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b181680c0805 namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 38b9ac95-d578-44e7-bd46-88a2f55446ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b181689b79e4 namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 1203409c-27a4-48a8-bfe9-96ff10cbe799 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:10:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:58Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b18173530ea8 namespace: kserve-ci-e2e-test resourceVersion: "25346" uid: f3ec2d1d-880b-4ebb-bb50-254ee67e8142 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b1919c196217 namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 7672c7d2-96db-4fc6-bdd4-cbf69d3758bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b1919c1c0053 namespace: kserve-ci-e2e-test resourceVersion: "25845" uid: 17c7530a-4ad3-4e63-86c9-abf7bdf889d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: daf37bea-70f1-4e36-9e83-a39881f93292 kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-07956-predictor-649f94cfd6-mgsrn.18c0b191fbc77034 namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 816eda07-2a4d-4e65-bc22-bd991ed240e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-07956-predictor-649f94cfd6 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 237c89de-883a-4b35-a8d8-d8a6f1a9e977 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: 'Created pod: success-200-isvc-07956-predictor-649f94cfd6-mgsrn' metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor-649f94cfd6.18c0b181443a4118 namespace: kserve-ci-e2e-test resourceVersion: "24998" uid: 59b65a7c-500a-433d-a6c9-6ca22f1d9fd3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "24975" uid: 2000124f-0a20-41f3-a8ed-83d1b80365d6 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Scaled up replica set success-200-isvc-07956-predictor-649f94cfd6 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-07956-predictor.18c0b18140a2998a namespace: kserve-ci-e2e-test resourceVersion: "24984" uid: 94a788b8-943a-400b-bec7-c88eb7490805 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: 2547c273-283d-4c09-a84a-5bda2dc35b34 kind: Event lastTimestamp: "2026-07-09T18:10:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:51Z" name: success-200-isvc-07956-predictor.18c0b184bfbbdda6 namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: db1e76c3-113d-4c18-a2bd-a2f9cbf7afed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:10:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: 2547c273-283d-4c09-a84a-5bda2dc35b34 kind: Event lastTimestamp: "2026-07-09T18:10:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:10:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:10:51Z" name: success-200-isvc-07956-predictor.18c0b184c0220e8b namespace: kserve-ci-e2e-test resourceVersion: "25296" uid: fc3c856b-d33c-449b-9274-9f33379bcd44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 2547c273-283d-4c09-a84a-5bda2dc35b34 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-07956-predictor.18c0b18bbe38024a namespace: kserve-ci-e2e-test resourceVersion: "25583" uid: 7732ffaa-d205-4a76-a9f5-c191599e7fc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-07956-predictor namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 2547c273-283d-4c09-a84a-5bda2dc35b34 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-07956-predictor.18c0b18bbea531c4 namespace: kserve-ci-e2e-test resourceVersion: "25590" uid: 1bff2971-fd39-4666-bf46-43dc631a8ea6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-07956 namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: 6b03f13d-2384-4d05-a61d-e33719e0b5f2 kind: Event lastTimestamp: "2026-07-09T18:11:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:08Z" name: success-200-isvc-07956.18c0b18140aa3065 namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: 8ac2f701-2138-4c94-bf9f-964b222e88e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-07956 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 6b03f13d-2384-4d05-a61d-e33719e0b5f2 kind: Event lastTimestamp: "2026-07-09T18:11:08Z" message: InferenceService [success-200-isvc-07956] is Ready metadata: creationTimestamp: "2026-07-09T18:11:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:08Z" name: success-200-isvc-07956.18c0b18c2f4653cf namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 678e7563-a725-4690-ac87-758a279f955e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:01:06.340533Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b100147ca3bf namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 144ab2f7-aabc-40ce-a915-eea64b7d05da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-14ef6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b10022dfad42 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: cc0ba1b3-1b13-41dd-bff4-583144203fb1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b100520fee41 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 55ba4460-4857-4928-9f91-b794319e2076 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b100539367fe namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 060f8ba1-33db-43f1-a070-7399ad18f5ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b100569c34c7 namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 2f0c849c-a6fa-41ec-be4d-280108f4a976 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b1005709510f namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 8e3755e0-815e-43c4-bdfd-77a374f88144 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b100571287d7 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: 1e3786b3-2ebd-4c32-a74d-640331897349 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b1005b4fb540 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 37452a75-2e89-4b8a-abd9-35eafff303e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:01:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:07Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b1005bbe8d65 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 784d1387-31a3-43ae-8507-d5ef40f40d4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T18:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:10:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:24Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b10093ca2362 namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: d384684c-86ca-4fb1-969b-f6befa486170 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b1813bda76ed namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 72111615-3406-4f79-bf1a-7e98c5dd4217 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:10:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:21Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b1813bdb0153 namespace: kserve-ci-e2e-test resourceVersion: "24960" uid: c0804ee6-7eee-4f0b-a8af-80792264cee2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: ef22e18a-ec83-476f-b7ed-d4577bb5c748 kind: Event lastTimestamp: "2026-07-09T18:10:24Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:10:24Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m.18c0b182083964b7 namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: e668d4f1-ef42-45aa-b7b2-10c10d01c077 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-14ef6-predictor-57b7ffcc88 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: c77b3f0b-c822-4906-944d-6547891125a0 kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'Created pod: success-200-isvc-14ef6-predictor-57b7ffcc88-b7n8m' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-14ef6-predictor-57b7ffcc88.18c0b100140e5824 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 57ac1e5f-fa15-45fd-a0ac-661107e435f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 2b32120a-e6ee-4fbd-b0b7-d36ff9be8e9e kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Scaled up replica set success-200-isvc-14ef6-predictor-57b7ffcc88 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-14ef6-predictor.18c0b100127b321b namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: eeb814e2-b999-47bb-a71b-4a51b0400f90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 26b681e7-6180-41e7-853b-20143ff51505 kind: Event lastTimestamp: "2026-07-09T18:01:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:36Z" name: success-200-isvc-14ef6-predictor.18c0b103919eff96 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 42cb5cb8-337c-473c-9bb5-52d389b81442 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 26b681e7-6180-41e7-853b-20143ff51505 kind: Event lastTimestamp: "2026-07-09T18:01:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:36Z" name: success-200-isvc-14ef6-predictor.18c0b103928b20f5 namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 77440ffe-3493-4963-8159-dc682b9d41bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 26b681e7-6180-41e7-853b-20143ff51505 kind: Event lastTimestamp: "2026-07-09T18:02:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:21Z" name: success-200-isvc-14ef6-predictor.18c0b10a914ae3ad namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 2c1a4cb6-3ca2-4b12-adec-2103e2cb2a73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14ef6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 26b681e7-6180-41e7-853b-20143ff51505 kind: Event lastTimestamp: "2026-07-09T18:02:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:21Z" name: success-200-isvc-14ef6-predictor.18c0b10a91b855cf namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: a7635f85-b035-4af9-8de5-c54664517123 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 2e4fb047-14fa-42e1-8c23-fb21a4811d4a kind: Event lastTimestamp: "2026-07-09T18:01:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:54Z" name: success-200-isvc-14ef6.18c0b100128555fb namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: bbff8db4-7fb7-464f-8484-d483760dc0c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 2e4fb047-14fa-42e1-8c23-fb21a4811d4a kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-14ef6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-14ef6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-14ef6.18c0b10013067a4c namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: f76d3093-a9af-4536-9886-ff758c55f29f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 2e4fb047-14fa-42e1-8c23-fb21a4811d4a kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-14ef6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-14ef6.18c0b1001306bec7 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: c2a1660b-b1e3-4612-ac48-4cffa0583c1a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14ef6 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 2e4fb047-14fa-42e1-8c23-fb21a4811d4a kind: Event lastTimestamp: "2026-07-09T18:01:54Z" message: InferenceService [success-200-isvc-14ef6] is Ready metadata: creationTimestamp: "2026-07-09T18:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:54Z" name: success-200-isvc-14ef6.18c0b10b50cf0edb namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: bbee7244-4f58-424e-9153-423351b3921b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:00:56.162938Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdb5dabc44 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 2d78e789-6c58-4b5a-bb74-01ec7a216dd2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-53a2e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdbd92c3e4 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 92ead5fb-c3e1-4253-bd9b-f64216732183 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdf398e741 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 7746b870-19fa-4642-9da0-d1a487d52df6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdf51ada43 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 0f787f2a-0ce9-472a-a94f-1546506b0d9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdf8599fb4 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 7ffd468b-09e6-40d6-9905-2258bbf1b741 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdf8c93d5d namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 09714b2c-f294-43fc-8b67-f0928957b705 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdf8d40f54 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: b4a43b0c-30cd-499a-8e4e-9e9c6a0f65ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdfbec0dd5 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: c7beb633-6bc6-4b0b-ae36-2ec78c48ae5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fdfc58a239 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: a624e878-f85d-4b5b-a1ef-ac046fe56d34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:33Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b0fe000f641c namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: e6061532-e3b7-4b9d-b51e-83d9212c9580 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b10e0f9bface namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: eebcb42d-a910-465a-8456-c06ab498cbda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b10e0f9f1c61 namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: f2546d58-d934-410a-b47e-9ea5157ea678 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 69678cc8-3f8d-4d11-a05c-fcc8866d34f5 kind: Event lastTimestamp: "2026-07-09T18:02:08Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:08Z" name: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv.18c0b10e88662978 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: d4008386-3125-4cc2-9eb7-9fb5d68e235d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-53a2e-predictor-57fccffd9b namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 59295955-b5fb-4444-9527-482d88757f8a kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Created pod: success-200-isvc-53a2e-predictor-57fccffd9b-6ljrv' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-53a2e-predictor-57fccffd9b.18c0b0fdb52cae38 namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 61a660b6-f850-4d23-bb09-bb4c2b930a04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 9b37dcc1-0966-47de-91f5-c03886fd9561 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Scaled up replica set success-200-isvc-53a2e-predictor-57fccffd9b from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-53a2e-predictor.18c0b0fdb2b25f49 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 2520a3e5-704a-4a1f-a61d-8f352c5e830f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 66f135ae-8a58-4837-8e60-872b435fb6dd kind: Event lastTimestamp: "2026-07-09T18:01:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:41Z" name: success-200-isvc-53a2e-predictor.18c0b101315f4216 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 09d6a2bb-a41c-437b-8511-c0fcfa94c432 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 66f135ae-8a58-4837-8e60-872b435fb6dd kind: Event lastTimestamp: "2026-07-09T18:01:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:41Z" name: success-200-isvc-53a2e-predictor.18c0b101319f1e66 namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 7e8f5ffa-854c-4213-ad08-7eff0e674bb9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 66f135ae-8a58-4837-8e60-872b435fb6dd kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:56Z" name: success-200-isvc-53a2e-predictor.18c0b104b04f02f2 namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: d992ae61-239b-43c5-bcf6-51ffb2dc5390 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 66f135ae-8a58-4837-8e60-872b435fb6dd kind: Event lastTimestamp: "2026-07-09T18:01:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:56Z" name: success-200-isvc-53a2e-predictor.18c0b104b0cbda19 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: 960ab352-74d9-457f-bbbc-1502d3e70c86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 1fea24b6-fae2-4616-b391-c12cb5a47a27 kind: Event lastTimestamp: "2026-07-09T18:01:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:43Z" name: success-200-isvc-53a2e.18c0b0fdb30906f0 namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 63878292-bfaf-435a-89be-6499222f15e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 1fea24b6-fae2-4616-b391-c12cb5a47a27 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-53a2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-53a2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-53a2e.18c0b0fdb3fce462 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9f4f18cd-ee21-406a-b9c4-2932b8a17050 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 1fea24b6-fae2-4616-b391-c12cb5a47a27 kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-53a2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-53a2e.18c0b0fdb3fd43b3 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 360e0478-29e7-47ee-8631-63727141badd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a2e namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 1fea24b6-fae2-4616-b391-c12cb5a47a27 kind: Event lastTimestamp: "2026-07-09T18:01:43Z" message: InferenceService [success-200-isvc-53a2e] is Ready metadata: creationTimestamp: "2026-07-09T18:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:43Z" name: success-200-isvc-53a2e.18c0b108bc92c5c5 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: d6c37262-739d-4397-8785-4ba048ba3797 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:11:31.415954Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-55ba8-predictor-65bd48468b-glbjd to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b1919de2991e namespace: kserve-ci-e2e-test resourceVersion: "25854" uid: bd1a0fc3-38d5-4b86-9613-a8d89de6f6ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-55ba8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191aa58bbb4 namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: 51f5b2bc-4eda-4cd3-9efd-cecf28257759 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25858" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191ddaa1f54 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 7d29266a-1218-4ee9-90c3-60cbcc2b75cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191df68e39d namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: b83b1e82-df04-4cc0-83ee-c451ad1e040f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191e437c499 namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: e70de43d-081e-46b4-8f6b-049dc14407b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191e4aec966 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: 785a8d3b-cd41-4b74-bbf1-146e12de99a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191e4b72d9f namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: 8d6b1ecd-3dde-4b29-aeb5-08506198bb17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191e7575071 namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: 5592be26-7886-4de0-b124-f01d47bd81c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:11:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:32Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b191e7ea9e4e namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: f3d32a25-be7f-470b-8f73-1bdae9f3cd71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:12:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:09Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b1921156be72 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: b541eae2-85cb-4868-bf04-89b30e9d308e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:20:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:20:46Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b212c2302ba7 namespace: kserve-ci-e2e-test resourceVersion: "30334" uid: 34ee90cf-b178-4dae-a9ce-46edddb272ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:20:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: da32fe16-b3e2-45c6-b20b-429e1eb9dabd kind: Event lastTimestamp: "2026-07-09T18:20:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:20:46Z" name: success-200-isvc-55ba8-predictor-65bd48468b-glbjd.18c0b212c230a879 namespace: kserve-ci-e2e-test resourceVersion: "30335" uid: a0ed7ccd-0e6a-47d5-acf4-1bb0a7f7bf76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-55ba8-predictor-65bd48468b namespace: kserve-ci-e2e-test resourceVersion: "25837" uid: 15b60045-2327-4aff-aa40-89a8bd1940a7 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'Created pod: success-200-isvc-55ba8-predictor-65bd48468b-glbjd' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-55ba8-predictor-65bd48468b.18c0b1919d5aa845 namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: 23aa7dbe-106e-4357-9d00-9cc4e8b4a3c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 8ee8b539-7a68-444f-b720-be65a2bcf06f kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: Scaled up replica set success-200-isvc-55ba8-predictor-65bd48468b from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-55ba8-predictor.18c0b1919b8fad76 namespace: kserve-ci-e2e-test resourceVersion: "25840" uid: 723271bd-688a-44ee-b290-4f0f245b15e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 8f9d9f28-93f0-43d3-93df-ac81a3cc64ba kind: Event lastTimestamp: "2026-07-09T18:12:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:01Z" name: success-200-isvc-55ba8-predictor.18c0b1951a133260 namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: 0183b4f4-2285-4375-afcf-0db9fbe5f502 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 8f9d9f28-93f0-43d3-93df-ac81a3cc64ba kind: Event lastTimestamp: "2026-07-09T18:12:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:01Z" name: success-200-isvc-55ba8-predictor.18c0b1951a7a0ccb namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: 874bac0a-6b5d-4d37-ad10-884db8e43fee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:12:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 8f9d9f28-93f0-43d3-93df-ac81a3cc64ba kind: Event lastTimestamp: "2026-07-09T18:12:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:46Z" name: success-200-isvc-55ba8-predictor.18c0b19c183681d7 namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 4d587c9f-52d4-4b84-9a53-e55eea2ef51f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:12:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-55ba8-predictor namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 8f9d9f28-93f0-43d3-93df-ac81a3cc64ba kind: Event lastTimestamp: "2026-07-09T18:12:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:46Z" name: success-200-isvc-55ba8-predictor.18c0b19c189d1ec9 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: 2068b9a6-4f2f-4038-b2d4-1df610dcd77c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 9f8ca9c3-e019-4382-b771-2a17a3dce0f5 kind: Event lastTimestamp: "2026-07-09T18:12:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:19Z" name: success-200-isvc-55ba8.18c0b1919bc044a5 namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: b0bd49a8-447e-4a48-bba3-633abb6104b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 9f8ca9c3-e019-4382-b771-2a17a3dce0f5 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-55ba8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-55ba8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-55ba8.18c0b1919c2e1448 namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: b7fc8c43-a92d-48c7-87ea-ef2e6be06283 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 9f8ca9c3-e019-4382-b771-2a17a3dce0f5 kind: Event lastTimestamp: "2026-07-09T18:11:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-55ba8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:11:31Z" name: success-200-isvc-55ba8.18c0b1919c2e793f namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: 35dab951-3cde-4571-82a5-a02fd1a2a7e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-55ba8 namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: 9f8ca9c3-e019-4382-b771-2a17a3dce0f5 kind: Event lastTimestamp: "2026-07-09T18:12:19Z" message: InferenceService [success-200-isvc-55ba8] is Ready metadata: creationTimestamp: "2026-07-09T18:12:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:19Z" name: success-200-isvc-55ba8.18c0b19cce8b9b77 namespace: kserve-ci-e2e-test resourceVersion: "26657" uid: 21dd9244-dd72-4f40-a9cc-3d004e6a2b8b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:59:56.096281Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efb99a5800 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 5b41b798-be3c-400e-b8e8-b2260fe0844a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efd3b5d979 namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: e5fa569c-5eb0-4840-8a2f-de9959a563ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efd51ed40a namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 547a0210-6e84-400d-964d-0de39201e67f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efd7f83c64 namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 0effa6e2-ac7f-4f99-95a5-790bef585465 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efd86f89c4 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 8647f0c3-2da8-4918-bb08-0a8a362eb631 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efd87936e4 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 49168445-f870-4ed7-8c6d-5db1fa3ecbc8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efdd3dd5c5 namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: 69010deb-4014-4482-9629-3b590ba9b5a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0efddb43263 namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 7b868765-09f0-416c-afa4-8e27d0368777 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T18:00:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:33Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b0eff6c17157 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 00181b25-a542-49ba-8bd0-c2c67cb64289 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b100117f94fc namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 8737eb70-1a88-4b7d-9e50-4df3b894a95e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T18:01:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:01:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:06Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b10011806419 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: d5780278-10de-4894-b581-bc7b5d997df0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 0f6f781e-b38d-4923-90dc-b95195ad9adb kind: Event lastTimestamp: "2026-07-09T18:01:08Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:08Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz.18c0b1007f380198 namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 15f1e437-0ad0-42c2-a303-38bbd1e07f5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5bd2e-predictor-66fc6dbb48 namespace: kserve-ci-e2e-test resourceVersion: "19655" uid: c05e9c34-6025-4087-a37d-7f6955cf88cf kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'Created pod: success-200-isvc-5bd2e-predictor-66fc6dbb48-nndpz' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor-66fc6dbb48.18c0b0efb921317a namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: eeacc9da-836a-4207-84ec-44fc7309edae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: ab68f861-f0f8-485c-9e47-b5da6f9befce kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Scaled up replica set success-200-isvc-5bd2e-predictor-66fc6dbb48 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e-predictor.18c0b0efb528dbd4 namespace: kserve-ci-e2e-test resourceVersion: "19660" uid: 653f9421-e1d0-481d-8cf3-3a8323c4dd69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 3e28ab87-632d-434f-9fad-30dce60726da kind: Event lastTimestamp: "2026-07-09T18:00:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:41Z" name: success-200-isvc-5bd2e-predictor.18c0b0f3343cf72d namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 94a0c4ea-daaf-48ea-9158-a8f6b5f94232 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 3e28ab87-632d-434f-9fad-30dce60726da kind: Event lastTimestamp: "2026-07-09T18:00:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:41Z" name: success-200-isvc-5bd2e-predictor.18c0b0f334d16953 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 2534a8a1-43a1-487f-96a8-e8de62e40b07 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 3e28ab87-632d-434f-9fad-30dce60726da kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-5bd2e-predictor.18c0b0f6b386f532 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: de8a2442-1b2e-49de-a1da-92be44d85a6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bd2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 3e28ab87-632d-434f-9fad-30dce60726da kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-5bd2e-predictor.18c0b0f6b3f6dcd8 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 817ad99a-f4bf-44b8-a632-8bf32b431ee0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: d9931192-4a70-4705-9a03-3c1eb918cbdb kind: Event lastTimestamp: "2026-07-09T18:00:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:43Z" name: success-200-isvc-5bd2e.18c0b0efb51fbdca namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: eb01373a-18ab-4bca-9e39-829b8a8bec12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: d9931192-4a70-4705-9a03-3c1eb918cbdb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-5bd2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5bd2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e.18c0b0efb5b454c0 namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: ad87b21b-081d-4015-a0dc-ec4111380880 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: d9931192-4a70-4705-9a03-3c1eb918cbdb kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5bd2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-5bd2e.18c0b0efb5b4b494 namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: d064a82a-f6c4-410e-bb51-41b168199dc3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bd2e namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: d9931192-4a70-4705-9a03-3c1eb918cbdb kind: Event lastTimestamp: "2026-07-09T18:00:43Z" message: InferenceService [success-200-isvc-5bd2e] is Ready metadata: creationTimestamp: "2026-07-09T18:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:43Z" name: success-200-isvc-5bd2e.18c0b0fab2e63e4e namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: f45b11db-c108-4693-8814-ce5ec12b2847 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:11:21.322218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25621" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5d567-predictor-69476f468b-r462g to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f44405d13 namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: 6c7e996b-77a4-4f25-83b3-4fcf1d78cef9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f5eb786c0 namespace: kserve-ci-e2e-test resourceVersion: "25679" uid: ea8cf7a7-e06f-4ff9-8726-afa45e50cc05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f60481c16 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 8e267801-90bc-4b83-b0a6-131ddf24ef59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f63aae9ce namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 26b34ef1-e44f-4984-84aa-34bbc800da11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f6424e1e5 namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 202630d5-a659-4e47-bc34-acb6183847a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f64314cc0 namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: e5da51a2-1e06-4593-989d-88824a319125 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f67b87560 namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: d2e941ee-1429-48ee-a49a-49d5793abfeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f684afe82 namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: d72c26be-63cb-42ff-bef5-389e839617f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T18:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:12:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:08Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b18f7d6887bc namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: cd9b2d42-a7c9-4da7-927f-e66a014d0e4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:28:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:28:50Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b28393216bd0 namespace: kserve-ci-e2e-test resourceVersion: "33698" uid: ae2a2db3-96f7-4ae8-be5e-b991c9ccd3f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:28:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:28:50Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b28393220fad namespace: kserve-ci-e2e-test resourceVersion: "33699" uid: f280efb6-e257-4e16-9535-2e26f7fd04bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5d567-predictor-69476f468b-r462g namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: a6e60ae5-af8c-4efa-857f-d2998e757b5d kind: Event lastTimestamp: "2026-07-09T18:28:53Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:28:53Z" name: success-200-isvc-5d567-predictor-69476f468b-r462g.18c0b284326fa266 namespace: kserve-ci-e2e-test resourceVersion: "33770" uid: e08fe2b9-3a35-4816-8679-ffc08d15c691 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5d567-predictor-69476f468b namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: 92d0131c-526f-40a3-80b3-0c9db47691f9 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: 'Created pod: success-200-isvc-5d567-predictor-69476f468b-r462g' metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor-69476f468b.18c0b18f43af9f41 namespace: kserve-ci-e2e-test resourceVersion: "25624" uid: 49a784e9-82eb-439b-ab7a-7adfa81d2a0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25594" uid: 0f474d9e-3700-47da-ad1f-f4b6c73fd71e kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Scaled up replica set success-200-isvc-5d567-predictor-69476f468b from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-5d567-predictor.18c0b18f40de0dbd namespace: kserve-ci-e2e-test resourceVersion: "25600" uid: 2b2f2526-f75f-4ab3-a078-84692f4d3d15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25596" uid: d757d690-795e-447a-b1c6-fa32d962daa5 kind: Event lastTimestamp: "2026-07-09T18:11:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:51Z" name: success-200-isvc-5d567-predictor.18c0b192c06b65b1 namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: a11565f5-d867-404d-a623-d7514ce3528e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25596" uid: d757d690-795e-447a-b1c6-fa32d962daa5 kind: Event lastTimestamp: "2026-07-09T18:11:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:11:51Z" name: success-200-isvc-5d567-predictor.18c0b192c16154f5 namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 4d4506f4-5c88-4bc5-8daa-7a04b5c9e121 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: d757d690-795e-447a-b1c6-fa32d962daa5 kind: Event lastTimestamp: "2026-07-09T18:12:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:36Z" name: success-200-isvc-5d567-predictor.18c0b199bec8ac3f namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 639d9d73-95d0-407e-8ccf-8e15f9a89cf6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T18:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5d567-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: d757d690-795e-447a-b1c6-fa32d962daa5 kind: Event lastTimestamp: "2026-07-09T18:12:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:36Z" name: success-200-isvc-5d567-predictor.18c0b199bf0ead73 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: ccd88d97-bb1a-4849-b458-414fd95a147d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5d567 namespace: kserve-ci-e2e-test resourceVersion: "25560" uid: 64b783e4-9b78-4762-a2a7-f5419f450c1d kind: Event lastTimestamp: "2026-07-09T18:12:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:18Z" name: success-200-isvc-5d567.18c0b18f40f6c799 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 8b66e08c-ae04-4599-a45c-632c6ab4efca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5d567 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 64b783e4-9b78-4762-a2a7-f5419f450c1d kind: Event lastTimestamp: "2026-07-09T18:12:18Z" message: InferenceService [success-200-isvc-5d567] is Ready metadata: creationTimestamp: "2026-07-09T18:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:18Z" name: success-200-isvc-5d567.18c0b19c8d2ccddc namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: 9d911e83-652a-4993-aa13-dc825bd2ab95 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:49:10.914709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-654f2-predictor-59b774c954-qhsv8 to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:49:10Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05981c8c5ea namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: a1d279df-69d5-46c5-8993-58f06017b39e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:49:11Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b0599bff5758 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: 4804e84f-1d8e-44c8-9d62-5abce0f34b9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:11Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" metadata: creationTimestamp: "2026-07-09T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:11Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b0599db3a1da namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: dc7c0bc2-2372-48b5-a747-8e350d164603 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" in 16.423s (16.423s including waiting). Image size: 1335612516 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05d7097d53e namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 90c78c03-62c3-48a4-b954-e2b420febe7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05d74820e28 namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: a8097233-1768-40e1-b0a7-0bb2925bb90e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05d74f9e71c namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 0b95177f-021c-479f-bf44-1fe99786a660 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T17:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:27Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05d7508000d namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: fe06d006-a01b-4aa5-850d-c263defe3090 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.54s (2.54s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:30Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05e0c7bfcd0 namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: a62b324f-7efc-4251-a306-0e2ebbe4dd81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:30Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05e102ec6b9 namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 777cf9df-b25b-4447-b747-db2d0b7bbf10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:49:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:30Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05e10bd6a99 namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: 5f8e70f3-346b-47fc-b86a-c043e63aea91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:50:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:07Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b05e43d70a8c namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: d670aac3-cf2b-42c1-8536-ce35b3fe6c2f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b06e86f6e20d namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: c2e13712-5172-4261-a13c-91e4286fc2c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b06e86f75c3f namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 3a97a0da-f24f-4428-a6c4-8a163a39853e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-654f2-predictor-59b774c954-qhsv8 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 3322d87e-2ead-4833-a380-8f5c1c97a1a3 kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-654f2-predictor-59b774c954-qhsv8.18c0b06ecc0a5c2b namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: 00aedf71-b008-40e7-9a29-2bafe7be288c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-654f2-predictor-59b774c954 namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 9a4bd243-5c90-455f-b515-7838feef2bd0 kind: Event lastTimestamp: "2026-07-09T17:49:10Z" message: 'Created pod: success-200-isvc-654f2-predictor-59b774c954-qhsv8' metadata: creationTimestamp: "2026-07-09T17:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:10Z" name: success-200-isvc-654f2-predictor-59b774c954.18c0b05981070c3d namespace: kserve-ci-e2e-test resourceVersion: "14188" uid: 20281e44-3397-4ec4-9815-5f8b26ab6455 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: fdf9d88a-f948-4265-b983-59f789d5ef13 kind: Event lastTimestamp: "2026-07-09T17:49:10Z" message: Scaled up replica set success-200-isvc-654f2-predictor-59b774c954 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:10Z" name: success-200-isvc-654f2-predictor.18c0b05979e48e41 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 77f6b9f1-aee5-447b-9a7e-7c8b4b0f1a6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: dcee6faf-ca56-4d80-8782-b5e556413a59 kind: Event lastTimestamp: "2026-07-09T17:50:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:10Z" name: success-200-isvc-654f2-predictor.18c0b05cff80ab9a namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 26bfdbd9-49d7-4e04-960b-b7e7d70f56e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: dcee6faf-ca56-4d80-8782-b5e556413a59 kind: Event lastTimestamp: "2026-07-09T17:50:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:10Z" name: success-200-isvc-654f2-predictor.18c0b05d00013a48 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 27a78ae1-4329-47c7-b0d6-5560ec9d4a7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: dcee6faf-ca56-4d80-8782-b5e556413a59 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-654f2-predictor.18c0b063fe3731c9 namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 15cd277b-fefc-48fa-959c-9b181b1f35a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-654f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: dcee6faf-ca56-4d80-8782-b5e556413a59 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-654f2-predictor.18c0b063fe9dd944 namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: 4808e35d-8561-4b95-929c-774d85038a34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-654f2 namespace: kserve-ci-e2e-test resourceVersion: "14167" uid: 60a9dbd7-4358-4c1c-9633-e4be2b01e36e kind: Event lastTimestamp: "2026-07-09T17:50:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:17Z" name: success-200-isvc-654f2.18c0b0597ff8f1e1 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: efad5781-fcca-4bd9-9a1f-da41b9f043c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-654f2 namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 60a9dbd7-4358-4c1c-9633-e4be2b01e36e kind: Event lastTimestamp: "2026-07-09T17:50:17Z" message: InferenceService [success-200-isvc-654f2] is Ready metadata: creationTimestamp: "2026-07-09T17:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:17Z" name: success-200-isvc-654f2.18c0b0690089d1fd namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 7d00dd05-1772-4270-b075-dd08d0c3be8d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:02:06.652836Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-67922-predictor-5948d8dd68-jknmx to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e1f6155d3 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: bf09983d-3f5f-4d0b-97bf-c1cf372044c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e397bad42 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 05147da5-3dc9-4300-8bdd-dc527632041e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e3adfedcc namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: d9228b21-baf1-498d-a5c5-07a386121856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e3e5ec471 namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: edc1c1f7-fd81-4629-ac18-49ff8bc37524 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e3ee0ad1b namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 0498ed99-ceb7-4137-84e4-26f866028d6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e3eeaa6b1 namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 9cb234c5-00cc-4691-9130-85eac01e23c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e42129506 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: b28af51a-561d-4070-bc5a-3bd33ca237f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e428ca73e namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 3dc041f8-873f-4217-b5bc-2558391533a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:11:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:23Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b10e6168aaa8 namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 76874743-ae14-46fc-9513-f4c4dab2dbe0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b18f40c89521 namespace: kserve-ci-e2e-test resourceVersion: "25599" uid: 4085fc94-6a42-4179-b20e-92eec13abf6e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:11:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:21Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b18f40c95542 namespace: kserve-ci-e2e-test resourceVersion: "25604" uid: e250b811-d8ab-4d60-a38a-cc5e312f9901 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67922-predictor-5948d8dd68-jknmx namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: a5bad1b4-d188-4ad0-9ef0-6b8ab06193d4 kind: Event lastTimestamp: "2026-07-09T18:11:23Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:11:23Z" name: success-200-isvc-67922-predictor-5948d8dd68-jknmx.18c0b18fd5f10400 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: a1c5d429-509d-4aea-9ac9-0d3cc7a039bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-67922-predictor-5948d8dd68 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: eae9effe-4820-480f-a8ab-ecd6f20428ec kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'Created pod: success-200-isvc-67922-predictor-5948d8dd68-jknmx' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-67922-predictor-5948d8dd68.18c0b10e1dfcced5 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 2bed2db4-ff0b-4839-b519-5cf7d4768782 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: fda75078-b6f4-42b5-9757-8033210c1b29 kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: Scaled up replica set success-200-isvc-67922-predictor-5948d8dd68 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-67922-predictor.18c0b10e167e0743 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: f6d6116c-7413-430d-878b-f3f921d820d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 6f150d0d-e04b-4e7f-bdd5-544131081fa1 kind: Event lastTimestamp: "2026-07-09T18:02:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:36Z" name: success-200-isvc-67922-predictor.18c0b11193fd04f3 namespace: kserve-ci-e2e-test resourceVersion: "21640" uid: d4564e75-7e2d-495c-a1ff-5c5715de7a7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 6f150d0d-e04b-4e7f-bdd5-544131081fa1 kind: Event lastTimestamp: "2026-07-09T18:02:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:36Z" name: success-200-isvc-67922-predictor.18c0b111945e80ab namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: ac872fe4-30a8-4e02-9d97-0a15ced60cd7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 6f150d0d-e04b-4e7f-bdd5-544131081fa1 kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:06Z" name: success-200-isvc-67922-predictor.18c0b11891b6d115 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: e8c74215-9e54-4ca9-9b77-ac33387a23ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67922-predictor namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 6f150d0d-e04b-4e7f-bdd5-544131081fa1 kind: Event lastTimestamp: "2026-07-09T18:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:02:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:06Z" name: success-200-isvc-67922-predictor.18c0b118921c12cd namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: 86809299-069a-4302-89db-d45b206ab0b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: fce81022-2eee-4691-a20b-3dbf84a0c84d kind: Event lastTimestamp: "2026-07-09T18:02:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:53Z" name: success-200-isvc-67922.18c0b10e15332ab7 namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: f0e703f5-e337-4350-ab45-ee4b090ff6e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: fce81022-2eee-4691-a20b-3dbf84a0c84d kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-67922": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-67922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-67922.18c0b10e18b14cdc namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 0739d499-1c81-45ca-8863-cce68f7ca477 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: fce81022-2eee-4691-a20b-3dbf84a0c84d kind: Event lastTimestamp: "2026-07-09T18:02:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-67922": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:06Z" name: success-200-isvc-67922.18c0b10e18b1b7c6 namespace: kserve-ci-e2e-test resourceVersion: "21303" uid: caa961d4-8801-47bb-adb1-008165c55309 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67922 namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: fce81022-2eee-4691-a20b-3dbf84a0c84d kind: Event lastTimestamp: "2026-07-09T18:02:53Z" message: InferenceService [success-200-isvc-67922] is Ready metadata: creationTimestamp: "2026-07-09T18:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:53Z" name: success-200-isvc-67922.18c0b1191d4883c8 namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 04127dfe-e665-473d-99f5-f522e39bda2d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:51:41.288925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-71ac7-predictor-6899d48d94-czfzn to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07c84c950ed namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: a2e7bf16-bc43-41c5-82bb-929f701fd0b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07c9e360cd6 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 2a2e4cb8-36fc-4a3c-a043-e8f624becafc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07c9fd560fd namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: fabd087d-75b3-4003-a7ad-ada446e167da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07ca2c12c9d namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 785c1078-5ffc-47b3-bb56-d6b87e53854b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07ca32c9887 namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 83a8a3fd-ae43-4ee9-af0b-de90dbf72dd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07ca336057d namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 44a77196-4a60-43ee-a395-c41737b7631e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07ca5c0f56a namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 769e95e5-50d1-444a-bd97-31bebb7367a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07ca64cbf29 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: e00bd5ec-e2ce-4cb3-b6d5-60b268569960 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b07ca9c91f50 namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: f171dfbc-e3fd-4d16-89cd-10960d5ed635 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b0fdb20c486a namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: c1779dbc-f181-4e05-b89b-27677a9e1170 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T18:00:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:56Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b0fdb20cd7f3 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 71b7d694-93df-49cc-9af8-528a8d1f17c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 9a229921-0ad5-4fc3-947f-4b0c2f8f029d kind: Event lastTimestamp: "2026-07-09T18:00:57Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:57Z" name: success-200-isvc-71ac7-predictor-6899d48d94-czfzn.18c0b0fe1e424706 namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: d28b1c39-5a32-4524-856f-f8d2d09f72d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-71ac7-predictor-6899d48d94 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: c9564ecf-4221-4e15-9731-4bc4811430f7 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'Created pod: success-200-isvc-71ac7-predictor-6899d48d94-czfzn' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor-6899d48d94.18c0b07c8433e5c8 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 7f9ef9ed-b3c3-42a1-81e3-ce449e1ecb81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: aaad7253-5372-487e-b41f-7470c300442c kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Scaled up replica set success-200-isvc-71ac7-predictor-6899d48d94 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:41Z" name: success-200-isvc-71ac7-predictor.18c0b07c7e5fa632 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: dafea1ee-b028-4a27-bca9-6c7d707ead9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: f8ce085f-db95-4837-8717-5ac7f7b8ee9f kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: success-200-isvc-71ac7-predictor.18c0b07ffccd6754 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: 01256256-9a10-4667-8461-d4b9070ea68e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: f8ce085f-db95-4837-8717-5ac7f7b8ee9f kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: success-200-isvc-71ac7-predictor.18c0b07ffd1f844e namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e71123c1-5b92-402b-8c91-868ec2a3aa79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:52:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: f8ce085f-db95-4837-8717-5ac7f7b8ee9f kind: Event lastTimestamp: "2026-07-09T17:52:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:56Z" name: success-200-isvc-71ac7-predictor.18c0b0837c7d0543 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: ab4be42b-3e34-4c7a-ac95-49b6ee1223fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:52:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-71ac7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: f8ce085f-db95-4837-8717-5ac7f7b8ee9f kind: Event lastTimestamp: "2026-07-09T17:52:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:56Z" name: success-200-isvc-71ac7-predictor.18c0b0837d003b26 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 9cac13cb-9dda-4ed6-810e-9a147a6aab0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: d376454e-7aaf-41f0-8042-7c9ecab867ca kind: Event lastTimestamp: "2026-07-09T17:52:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:28Z" name: success-200-isvc-71ac7.18c0b07c7e5e2c3b namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 02e63833-8363-4d7c-8996-8abc3b0cc7ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-71ac7 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: d376454e-7aaf-41f0-8042-7c9ecab867ca kind: Event lastTimestamp: "2026-07-09T17:52:28Z" message: InferenceService [success-200-isvc-71ac7] is Ready metadata: creationTimestamp: "2026-07-09T17:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:52:28Z" name: success-200-isvc-71ac7.18c0b087665d7929 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 3f243927-b717-46d3-abe6-1d4fdd11099c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:50:41.307283Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk to ip-10-0-136-100.ec2.internal metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06e8d9a03e5 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 7dab2b9c-489f-4405-af61-fef8f2dde68f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-785cd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06e97c22028 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 4b1bf5d7-b3eb-447d-93ea-4cfd56f0c851 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ecafaff41 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: 25e00710-2113-481a-9a42-8f775d8630f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ecc6dc207 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 73065b06-e0aa-4840-afe7-5d75ca19eff3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ed0220373 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 67f8a201-9976-4f8a-b700-e04ae10d8338 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ed08fa806 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: ab019f30-d75e-4e6d-8f5b-01d216ffe128 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ed099356b namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: bbdc3978-db46-4dc5-b9d1-a35652a45bc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ed4484564 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 5f2e28a4-7864-4c27-a93a-018b5ba87560 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:50:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:42Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06ed4c01364 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 68be42a2-8bb9-475f-88cf-92e8039d453e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T17:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:59:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:58Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b06eddaa5722 namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: 005b3338-fff8-4aa0-8b9a-d3320abbb6f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b0efb669b48a namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 3616405a-dc8e-4299-ae8a-5c39cdc951b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:59:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:56Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b0efb66a62dd namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: 5b090f04-4c76-48ba-9b52-6a191fe96326 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: e2d67e88-66fd-44f1-96fa-0849e42332ac kind: Event lastTimestamp: "2026-07-09T17:59:58Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:58Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk.18c0b0f05229e422 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: e9537c2e-4dc2-45c2-b8f8-9ae3f1fbf640 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-100.ec2.internal source: component: kubelet host: ip-10-0-136-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-785cd-predictor-7cbb69cdbb namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 14c675f9-dfd6-42c3-a232-ee1107276e18 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'Created pod: success-200-isvc-785cd-predictor-7cbb69cdbb-tmptk' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-785cd-predictor-7cbb69cdbb.18c0b06e8cece9f1 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 162502fb-fbd2-4561-989b-790df616086e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 05f806b8-b8ac-437f-98cd-878d6586d09b kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Scaled up replica set success-200-isvc-785cd-predictor-7cbb69cdbb from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-785cd-predictor.18c0b06e8a29f8a9 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 9e40a528-0359-4783-8dec-b0f225911a18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:50:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 2b24a957-3f12-4517-aa65-818efe826a4a kind: Event lastTimestamp: "2026-07-09T17:51:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:11Z" name: success-200-isvc-785cd-predictor.18c0b072091a81d5 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: b76ff72d-b46d-480d-8569-3c14b1aa4843 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:50:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 2b24a957-3f12-4517-aa65-818efe826a4a kind: Event lastTimestamp: "2026-07-09T17:51:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:11Z" name: success-200-isvc-785cd-predictor.18c0b07209b319a4 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 51c7e7f7-05a0-4b7c-bbd1-338aab7bc430 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:51:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 2b24a957-3f12-4517-aa65-818efe826a4a kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: success-200-isvc-785cd-predictor.18c0b07907474922 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 33ed726f-bee5-4b0f-b32f-a440ccf42349 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:51:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-785cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 2b24a957-3f12-4517-aa65-818efe826a4a kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: success-200-isvc-785cd-predictor.18c0b07907b27e23 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 96c94c21-8092-4c66-9ac2-113bc662f7bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 8e79d4d3-4c01-4d7e-a5b9-2b5522eeef49 kind: Event lastTimestamp: "2026-07-09T17:51:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:28Z" name: success-200-isvc-785cd.18c0b06e8a2fa4dd namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 52e10152-d419-41b3-b61c-2ce258bd57b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 8e79d4d3-4c01-4d7e-a5b9-2b5522eeef49 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-785cd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-785cd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-785cd.18c0b06e8ae06346 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 6fd39ca9-1b4a-4987-bc4d-fccb740b61cb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 8e79d4d3-4c01-4d7e-a5b9-2b5522eeef49 kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-785cd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:41Z" name: success-200-isvc-785cd.18c0b06e8ae0bbf6 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 3e88892a-4c06-4da1-9232-ee9db2c19cce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-785cd namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 8e79d4d3-4c01-4d7e-a5b9-2b5522eeef49 kind: Event lastTimestamp: "2026-07-09T17:51:28Z" message: InferenceService [success-200-isvc-785cd] is Ready metadata: creationTimestamp: "2026-07-09T17:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:28Z" name: success-200-isvc-785cd.18c0b0799dd5b41d namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 0fc241d9-640a-4b66-a0e2-72027b2fdeb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:12:31.342676Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-5d567-6799fc7b-xq5cl to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:12:31Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b19f91cbd012 namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: 3dfbfce8-db97-4b92-8e0b-42c9b9404087 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:12:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-5d567-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:32Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b19fa0886e3d namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: 05fbe5bf-954a-46cd-bbf8-49ec17972f10 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:12:33Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:12:33Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b1a0056f1b67 namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 076444a3-7908-4eab-b04a-fbd03765db11 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5d567} kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:12:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:33Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b1a006c2e9db namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 4b1e492a-24e3-4cb7-a558-6296f398c803 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5d567} kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:12:33Z" message: 'Created container: switch-graph-5d567' metadata: creationTimestamp: "2026-07-09T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:33Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b1a009eeca93 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: 70cf1b62-1cc8-492d-a197-4cdd0e17eb99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5d567} kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:12:33Z" message: Started container switch-graph-5d567 metadata: creationTimestamp: "2026-07-09T18:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:12:33Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b1a00a669c07 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 29526133-10e1-4607-8d70-60962251cbda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5d567} kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:28:50Z" message: Stopping container switch-graph-5d567 metadata: creationTimestamp: "2026-07-09T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:28:50Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b2838952bb53 namespace: kserve-ci-e2e-test resourceVersion: "33682" uid: 3e719877-9066-4d02-9a56-d484426105a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T18:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5d567} kind: Pod name: switch-graph-5d567-6799fc7b-xq5cl namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4d548b75-56ac-400b-9dc5-d4e8f576990e kind: Event lastTimestamp: "2026-07-09T18:29:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T18:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:29:15Z" name: switch-graph-5d567-6799fc7b-xq5cl.18c0b283969d40d6 namespace: kserve-ci-e2e-test resourceVersion: "33939" uid: bee6be25-0f29-445b-afef-63fb74ffb1ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-5d567-6799fc7b namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 1173abe6-b6bb-4a75-9f0d-32c276e740ee kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: 'Created pod: switch-graph-5d567-6799fc7b-xq5cl' metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:31Z" name: switch-graph-5d567-6799fc7b.18c0b19f915ef538 namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: ca43d8f1-e278-4e20-8593-992f6ecc250d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-5d567 namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 5bb23f08-29d1-46f4-8c00-27eeb736f182 kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: Scaled up replica set switch-graph-5d567-6799fc7b from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:12:31Z" name: switch-graph-5d567.18c0b19f906c0a27 namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: b054a11b-586e-4306-b6e8-87ae32108726 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:12:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5d567 namespace: kserve-ci-e2e-test resourceVersion: "26777" uid: 165919bc-5712-4c34-9dc3-47585de371c8 kind: Event lastTimestamp: "2026-07-09T18:12:31Z" message: InferenceGraph [switch-graph-5d567] is Ready metadata: creationTimestamp: "2026-07-09T18:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:12:31Z" name: switch-graph-5d567.18c0b19f9636c3f5 namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: 03ffac6a-3a7a-494b-ae8a-a45d05516ddf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:12:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-5d567 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: fdd12c2b-f5ef-423c-ad39-5207717154d3 kind: Event lastTimestamp: "2026-07-09T18:13:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:13:01Z" name: switch-graph-5d567.18c0b1a30ee63948 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: cdfed083-cef2-451b-98e6-5cd6cc57d264 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:12:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-5d567 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: fdd12c2b-f5ef-423c-ad39-5207717154d3 kind: Event lastTimestamp: "2026-07-09T18:13:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:13:01Z" name: switch-graph-5d567.18c0b1a30f478a40 namespace: kserve-ci-e2e-test resourceVersion: "27068" uid: d320bc75-765c-46e2-aafa-c84f178b53d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:50:30.941161Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-654f2-66fc497bb7-ndlcp to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T17:50:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:50:30Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06c23bb8a7e namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: f0274aac-0f17-4916-b669-8a94ad53413e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:50:31Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:50:31Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06c3dd3885d namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: a654f9a3-359d-431a-b12e-c1c500d1eb8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-654f2} kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:50:31Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" metadata: creationTimestamp: "2026-07-09T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:31Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06c3f2afed7 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 47a891af-78a7-41d7-9129-1245645f93cf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-654f2} kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:50:33Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" in 2.388s (2.388s including waiting). Image size: 218072678 bytes.' metadata: creationTimestamp: "2026-07-09T17:50:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:33Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06ccd8e654f namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 116186a4-40c8-4396-8d36-610310722bec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-654f2} kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:50:33Z" message: 'Created container: switch-graph-654f2' metadata: creationTimestamp: "2026-07-09T17:50:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:33Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06cd1146fb5 namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: fa844237-9aa1-4182-afd8-2243b9876106 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-654f2} kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:50:33Z" message: Started container switch-graph-654f2 metadata: creationTimestamp: "2026-07-09T17:50:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:33Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06cd18d95b0 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 79d462e5-a841-464f-b71e-afcaa727b7d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-654f2} kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:50:41Z" message: Stopping container switch-graph-654f2 metadata: creationTimestamp: "2026-07-09T17:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:41Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06e7e266973 namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 75bcc280-3a40-4a89-9d8d-f11b518a7e31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-654f2} kind: Pod name: switch-graph-654f2-66fc497bb7-ndlcp namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 956b5d9c-e403-422a-8c6b-eec583c709ec kind: Event lastTimestamp: "2026-07-09T17:51:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:10Z" name: switch-graph-654f2-66fc497bb7-ndlcp.18c0b06f7ade538e namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: a0e24c85-fc77-4e11-a366-64065d561fec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-654f2-66fc497bb7 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: b1e068eb-9fa2-4b1a-b3d5-5c736928a628 kind: Event lastTimestamp: "2026-07-09T17:50:30Z" message: 'Created pod: switch-graph-654f2-66fc497bb7-ndlcp' metadata: creationTimestamp: "2026-07-09T17:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:30Z" name: switch-graph-654f2-66fc497bb7.18c0b06c22eb129e namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 119ff3cf-ae95-46b2-866d-967ec5962aee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-654f2 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 8bebd07c-bba1-45f9-8576-5d39fe14d0e2 kind: Event lastTimestamp: "2026-07-09T17:50:30Z" message: Scaled up replica set switch-graph-654f2-66fc497bb7 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:50:30Z" name: switch-graph-654f2.18c0b06c21d50ec8 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: f1d59a0a-fb16-4abb-b6f8-e078837f9a49 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-654f2 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 8233cd86-add5-437e-b27a-daab5aa87a71 kind: Event lastTimestamp: "2026-07-09T17:50:31Z" message: InferenceGraph [switch-graph-654f2] is Ready metadata: creationTimestamp: "2026-07-09T17:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:50:31Z" name: switch-graph-654f2.18c0b06c294959c8 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 146adbde-26b3-4280-b8fd-469b2c7a5071 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:51:41.541692Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-785cd-7d89ff7948-6lr7k to ip-10-0-141-169.ec2.internal metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:51:41Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b07c93da3b05 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: d5fe484f-0ba9-4261-ac70-fb4c97b85218 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-68c7fc57f5-6hnl9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:51:41Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b07cae5045f4 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 61e47d34-30b1-4022-96c7-137d77545f41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-785cd} kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b07cafc3664f namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: b342d7bf-b84e-4153-a5e9-5bafd559e9d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-785cd} kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: 'Created container: switch-graph-785cd' metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b07cb3f5af45 namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 9049db27-cb8d-4467-9fc1-fc98d1e3eec4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-785cd} kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: "2026-07-09T17:51:42Z" message: Started container switch-graph-785cd metadata: creationTimestamp: "2026-07-09T17:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:42Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b07cb4733136 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 5c7bad55-46b1-44c9-886b-ed61d186740c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-785cd} kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: "2026-07-09T17:59:55Z" message: Stopping container switch-graph-785cd metadata: creationTimestamp: "2026-07-09T17:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:55Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b0efab4248f8 namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: b435a846-f580-4dbe-af06-0c3172d20aaf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-785cd} kind: Pod name: switch-graph-785cd-7d89ff7948-6lr7k namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 36ce38a9-32eb-4647-b155-e985518e6d07 kind: Event lastTimestamp: "2026-07-09T18:00:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:23Z" name: switch-graph-785cd-7d89ff7948-6lr7k.18c0b0f04c2f5a3c namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: 5bb7ab91-41d1-482e-90c8-2280d81426e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-169.ec2.internal source: component: kubelet host: ip-10-0-141-169.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-785cd-7d89ff7948 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: f1e25671-d1fb-42b6-8da2-e715b59f411e kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: 'Created pod: switch-graph-785cd-7d89ff7948-6lr7k' metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:41Z" name: switch-graph-785cd-7d89ff7948.18c0b07c92e27ee5 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: a23d7eb7-411b-4bfb-b1ab-62a878cc9055 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-785cd namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: bcaad63a-b172-4d0b-9efe-002015c735fb kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: Scaled up replica set switch-graph-785cd-7d89ff7948 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:41Z" name: switch-graph-785cd.18c0b07c9142a2d4 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: e1acbed1-bc67-473d-93ed-4a28792281d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-785cd namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 02f73470-bb54-4827-bc51-1e163341bb2c kind: Event lastTimestamp: "2026-07-09T17:51:41Z" message: InferenceGraph [switch-graph-785cd] is Ready metadata: creationTimestamp: "2026-07-09T17:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:41Z" name: switch-graph-785cd.18c0b07c996fb13d namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: c2067e46-27b7-441b-aef5-da9a59b02894 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-785cd namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 993825ff-e488-4919-9a87-ae7a0c7883fe kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: switch-graph-785cd.18c0b0800d9d48f6 namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 13b5ee37-a1ce-47f7-bf96-2df57fd56c6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-785cd namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 993825ff-e488-4919-9a87-ae7a0c7883fe kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:56Z" name: switch-graph-785cd.18c0b0800de67bf1 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 1619edf8-9879-4362-8cf0-434ec6eb3ead reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-785cd namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 993825ff-e488-4919-9a87-ae7a0c7883fe kind: Event lastTimestamp: "2026-07-09T17:52:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:11Z" name: switch-graph-785cd.18c0b0838c9da0fb namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 11a8d385-5524-4913-ac34-88f2451f7eeb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-785cd namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 993825ff-e488-4919-9a87-ae7a0c7883fe kind: Event lastTimestamp: "2026-07-09T17:52:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:11Z" name: switch-graph-785cd.18c0b0838d19932e namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 58e91ed9-439d-4ed8-8f0f-48b71b4a1a5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33956"