--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:39:03.660600Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-cec47-67f6fbdc44-f52tz to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:39:03Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe186212f1e5e namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: d5c66e2e-dbb9-4896-9ae8-859fec27f6b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:04Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: "2026-07-07T02:39:04Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:39:04Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe1863af89ed7 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 62c635b2-eb32-4d85-8e44-22d2d181772b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cec47} kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: "2026-07-07T02:39:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:04Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe1863c3dae02 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: c89a6123-5d23-4e4b-9d16-a4b93a8da77c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cec47} kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: "2026-07-07T02:39:04Z" message: 'Created container: ensemble-graph-cec47' metadata: creationTimestamp: "2026-07-07T02:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:04Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe1863fc9e4fb namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: 5d67f3c1-6703-437b-8c61-d3c6f32fdaa2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cec47} kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: "2026-07-07T02:39:04Z" message: Started container ensemble-graph-cec47 metadata: creationTimestamp: "2026-07-07T02:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:04Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe186403c6c69 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 9def0c05-8609-4389-8e1e-5fac5d3f0202 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cec47} kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Stopping container ensemble-graph-cec47 metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:13Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe1887989deac namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 331f8cfb-1448-43d9-814f-853fb9d990d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cec47} kind: Pod name: ensemble-graph-cec47-67f6fbdc44-f52tz namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 50f3dd6b-32b0-410e-b1fa-bda78c95fe20 kind: Event lastTimestamp: "2026-07-07T02:39:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:40Z" name: ensemble-graph-cec47-67f6fbdc44-f52tz.18bfe188d5ef0322 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 05d09458-25dd-4f30-806c-fe01a2b40e29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-cec47-67f6fbdc44 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 82c17156-152b-4a10-aefa-c2c6feaaa568 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'Created pod: ensemble-graph-cec47-67f6fbdc44-f52tz' metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: ensemble-graph-cec47-67f6fbdc44.18bfe18620c30453 namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 569b594c-a1a4-45f0-9029-ca3d40da8e95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 3a4956c6-3ace-41fd-bcef-5d8a3e7dde83 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: Scaled up replica set ensemble-graph-cec47-67f6fbdc44 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: ensemble-graph-cec47.18bfe1861fbe6d13 namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 1cba2ea5-549a-4aba-8ea7-328cbeb742d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 90517ebf-0493-4c75-8208-758d306bc080 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: InferenceGraph [ensemble-graph-cec47] is Ready metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:03Z" name: ensemble-graph-cec47.18bfe186254a08d8 namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 3d07f063-79cb-49f3-88c4-b3527d131780 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:40:14.004115Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-e4a8c-d5b8f5585-rcdfh to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:40:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:40:14Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe19681fbec88 namespace: kserve-ci-e2e-test resourceVersion: "21748" uid: 2d34807e-3df4-4b31-aa15-8ad65180067b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:14Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:40:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-e4a8c-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:14Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe1968dbc036a namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 9c1ab3bf-dc59-4a0c-b3a0-3657e5dc1f49 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:15Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:40:15Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:40:15Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe196bf323a45 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: d356d3ae-39de-4275-8ea0-8977178091c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e4a8c} kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:40:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:15Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe196c0624813 namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 46945de8-c7f1-4166-89bb-93f91d98809d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e4a8c} kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:40:15Z" message: 'Created container: ensemble-graph-e4a8c' metadata: creationTimestamp: "2026-07-07T02:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:15Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe196c332815c namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: 7dc258a3-beef-42e5-9ade-8b25a065d3cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e4a8c} kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:40:15Z" message: Started container ensemble-graph-e4a8c metadata: creationTimestamp: "2026-07-07T02:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:15Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe196c3a2fba7 namespace: kserve-ci-e2e-test resourceVersion: "21773" uid: d7f9cb59-26f6-41f7-9568-cc0e41764ad3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e4a8c} kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Stopping container ensemble-graph-e4a8c metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:28Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe209b3350846 namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: 09053255-4f91-4bf0-ac7e-1e555932d1b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e4a8c} kind: Pod name: ensemble-graph-e4a8c-d5b8f5585-rcdfh namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 1dc200ee-7d9d-43db-8135-ec9e90d38abe kind: Event lastTimestamp: "2026-07-07T02:48:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:56Z" name: ensemble-graph-e4a8c-d5b8f5585-rcdfh.18bfe20a5a4fe504 namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: 4b4543f3-8bd7-47ed-8e0d-c034d2c84eda reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-e4a8c-d5b8f5585 namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 3f354158-262c-48be-b262-c6848c2ff57b kind: Event lastTimestamp: "2026-07-07T02:40:13Z" message: 'Created pod: ensemble-graph-e4a8c-d5b8f5585-rcdfh' metadata: creationTimestamp: "2026-07-07T02:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:13Z" name: ensemble-graph-e4a8c-d5b8f5585.18bfe19681809d28 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 94abcf6f-17a1-4c8a-aa64-27e84cc095a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: c448dc0d-12ba-4c9d-858d-0540423380e0 kind: Event lastTimestamp: "2026-07-07T02:40:13Z" message: Scaled up replica set ensemble-graph-e4a8c-d5b8f5585 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:13Z" name: ensemble-graph-e4a8c.18bfe19680a6f154 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: ada5afd5-4737-40f5-b28f-bc2bedc2d7de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: bbcd6dc3-96d9-406d-a4b9-2f9b7485713f kind: Event lastTimestamp: "2026-07-07T02:40:14Z" message: InferenceGraph [ensemble-graph-e4a8c] is Ready metadata: creationTimestamp: "2026-07-07T02:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:14Z" name: ensemble-graph-e4a8c.18bfe19685b6c2e6 namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 4069818b-9f47-472e-8da6-4e796e83037c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: dc6da1e2-89c6-4fea-a0fc-29f8fa839fc6 kind: Event lastTimestamp: "2026-07-07T02:40:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:44Z" name: ensemble-graph-e4a8c.18bfe199ff1d5817 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 79d2d344-aadc-4d61-9701-e7b2f582bd09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: dc6da1e2-89c6-4fea-a0fc-29f8fa839fc6 kind: Event lastTimestamp: "2026-07-07T02:40:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:44Z" name: ensemble-graph-e4a8c.18bfe199ff82d4ba namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: f7619a28-239b-454c-b16b-0e8e77b1901a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:26:58.554527Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2346e-predictor-7bc756bd48-sbb2x to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:26:58Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0dd4d7e7bf2 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: 3a003b15-a15f-44ec-ad70-21b7211e56e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:26:59Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0dd681fda1e namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: ddd808b9-c44d-4094-9ad8-5bfbfd6ab467 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:59Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0dd69b4325a namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 8fc19482-6c4d-468b-a51c-d4500722b9d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:11Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" in 12.431s (12.431s including waiting). Image size: 1334693976 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:11Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e04ea80c72 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: f79c0f52-8892-4387-9c95-2ba95f860967 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:11Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e051ccbcaf namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 95aab7bd-7c13-4a8f-a958-74056ff12d51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:11Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e05246d173 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 1c882232-d273-4387-8021-9fae1daedfc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:11Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T02:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:11Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e052558275 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: c94870cf-3386-4ec6-9dbc-2ea74bba54a9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.371s (2.371s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:13Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e0dfb70d3c namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: d6a3ca8e-1acb-42c7-b20c-a5b5367c1bdb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:13Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e0e29575d2 namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 3992595f-50af-4d5f-9968-0c8f4cf45b89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:27:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:13Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e0e312d39c namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 9e866144-8459-4cdd-b43f-d8905bbab99b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:28:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:41Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0e1507abf9b namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 2a1f3edb-e67c-48d1-9d6b-5a60369ca8a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0f4ab5d58ff namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: d1b02283-b494-47b1-8331-449924729d6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0f4ab5e36b5 namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 532520d4-6637-4f1e-ad08-c2374ca68c95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: da19c8dc-a81d-4173-98ba-92d4e1cc1dea kind: Event lastTimestamp: "2026-07-07T02:28:41Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:41Z" name: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x.18bfe0f556b873f1 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 6f9d33f7-de87-4cea-9bb9-896fe2bfb912 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2346e-predictor-7bc756bd48 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: c1706277-c6b9-495c-8901-864565ec1f95 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: 'Created pod: error-404-isvc-2346e-predictor-7bc756bd48-sbb2x' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: error-404-isvc-2346e-predictor-7bc756bd48.18bfe0dd4d169d99 namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: c0d37d3a-2573-4f54-aff8-fb172484ff07 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: 0ceec990-a170-4f5f-81cb-340f9f4ff949 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Scaled up replica set error-404-isvc-2346e-predictor-7bc756bd48 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: error-404-isvc-2346e-predictor.18bfe0dd4a3121ef namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: ced5ae70-cd34-48e9-af4a-2c301ce904ce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 381f3546-6be9-4f67-8571-345486b8cbcd kind: Event lastTimestamp: "2026-07-07T02:27:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:43Z" name: error-404-isvc-2346e-predictor.18bfe0e0c8ec9d6e namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 49760308-cc45-4d38-944e-fac236fae6b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 381f3546-6be9-4f67-8571-345486b8cbcd kind: Event lastTimestamp: "2026-07-07T02:27:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:43Z" name: error-404-isvc-2346e-predictor.18bfe0e0c974e69d namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 59a01758-6c1a-411d-a808-6f55d76697fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 381f3546-6be9-4f67-8571-345486b8cbcd kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:28Z" name: error-404-isvc-2346e-predictor.18bfe0eb46f06674 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 836036af-d8af-4609-a874-48ae18188da4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 381f3546-6be9-4f67-8571-345486b8cbcd kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:28Z" name: error-404-isvc-2346e-predictor.18bfe0eb477ae476 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: a96ae93d-59e7-40ea-a330-dccfa7e635da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2346e namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: 77cb6ba6-f4df-4fa9-aa5d-47bb7969ef6a kind: Event lastTimestamp: "2026-07-07T02:28:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:01Z" name: error-404-isvc-2346e.18bfe0dd4a35bcc9 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: a445faf4-71a7-4d95-8636-e1bd31043665 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2346e namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 77cb6ba6-f4df-4fa9-aa5d-47bb7969ef6a kind: Event lastTimestamp: "2026-07-07T02:28:01Z" message: InferenceService [error-404-isvc-2346e] is Ready metadata: creationTimestamp: "2026-07-07T02:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:01Z" name: error-404-isvc-2346e.18bfe0ec0ec0edc0 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 94199159-6111-4970-99dd-79aacfffa9ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:28:38.927472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3ea7f-predictor-8b8785f68-xf652 to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4ac300ff1 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: fa0b007a-b419-4832-bffa-c11a047551bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3ea7f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4b669b88d namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: c7a9a61c-e458-4dc3-984a-4349a9ae0c3a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:28:39Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4e99a45e5 namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: 985406fe-dc37-4bfa-bc43-68b4c94c574d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4eb3d9e0b namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 49bfc6a6-902f-45cc-92c5-2493c027c4b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:40Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4ee255316 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 4ea0fecb-658a-45c1-af9d-1572be0fca4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:40Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4eeb93131 namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: aada7192-c9e7-4100-b0a3-43fc3d9301ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:40Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4eec414e9 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: c1c6b321-f005-4fc6-85d4-94a83a3071c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:40Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4f1cf1408 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: bfdbc646-1395-4387-a1af-7b19ed8c5f59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:40Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f4f298aef8 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: d4b207f8-c640-47e3-a0b2-7b66fd495678 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:29:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:17Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe0f52f8c3378 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 4eca220b-330e-42b6-91f6-d6bee72610f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe17823d42667 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 2369b093-4e29-4469-ad97-ef02a6f3500b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: db1e761e-49c2-4e82-aae2-de9d61839988 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-3ea7f-predictor-8b8785f68-xf652.18bfe17823d54d8e namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 73e5e67b-5945-4246-b67e-d6853b486a94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3ea7f-predictor-8b8785f68 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: a548d072-df79-4c53-b31a-93ddb2fce4bf kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: 'Created pod: error-404-isvc-3ea7f-predictor-8b8785f68-xf652' metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-3ea7f-predictor-8b8785f68.18bfe0f4ab457148 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: bab06e55-6942-467a-9bb4-c0a78b21091c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: b4ccb3a3-0708-4b42-bee9-1179e7aff11c kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Scaled up replica set error-404-isvc-3ea7f-predictor-8b8785f68 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-3ea7f-predictor.18bfe0f4a6aa5dbf namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 32c97b44-8c46-4876-a696-3ee048785b30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: db8b8fb4-ad79-4055-8479-af27060f7fba kind: Event lastTimestamp: "2026-07-07T02:29:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:23Z" name: error-404-isvc-3ea7f-predictor.18bfe0f824f02202 namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 9f9cf384-a8f3-4bfd-adc5-3ace5aa4a258 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: db8b8fb4-ad79-4055-8479-af27060f7fba kind: Event lastTimestamp: "2026-07-07T02:29:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:23Z" name: error-404-isvc-3ea7f-predictor.18bfe0f825980029 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 8c9fc5c5-dbc4-4ff7-bbed-3375f6ae69d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: db8b8fb4-ad79-4055-8479-af27060f7fba kind: Event lastTimestamp: "2026-07-07T02:29:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:53Z" name: error-404-isvc-3ea7f-predictor.18bfe0fba45980e7 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: c5146531-38d8-41e5-955c-be779e6e762b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: db8b8fb4-ad79-4055-8479-af27060f7fba kind: Event lastTimestamp: "2026-07-07T02:29:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:53Z" name: error-404-isvc-3ea7f-predictor.18bfe0fba4c299dc namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 2f693256-2ee1-45b7-b9c3-d1f326732404 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 85cbb6a7-6e71-4940-8332-831e1c2a68b4 kind: Event lastTimestamp: "2026-07-07T02:29:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:27Z" name: error-404-isvc-3ea7f.18bfe0f4a6dc2e98 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: bf11d5c1-b8b0-4cc8-a1b7-f36689de8878 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 85cbb6a7-6e71-4940-8332-831e1c2a68b4 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: 'Failed to update status for InferenceService "error-404-isvc-3ea7f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3ea7f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-3ea7f.18bfe0f4a8e8a5d1 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 2775f698-b989-44a6-bbc5-291e8c54e67a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 85cbb6a7-6e71-4940-8332-831e1c2a68b4 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3ea7f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:38Z" name: error-404-isvc-3ea7f.18bfe0f4a8e90337 namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 7e30d2d8-f37a-405a-8fc4-44807513310b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 85cbb6a7-6e71-4940-8332-831e1c2a68b4 kind: Event lastTimestamp: "2026-07-07T02:29:27Z" message: InferenceService [error-404-isvc-3ea7f] is Ready metadata: creationTimestamp: "2026-07-07T02:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:27Z" name: error-404-isvc-3ea7f.18bfe0ffeb0214f9 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 1df9fb22-ad49-467d-8ff9-cbb8dd7acceb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:48:29.031185Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25579" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4831c-predictor-5944766686-tvxdv to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:48:29Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe209c3e58ce3 namespace: kserve-ci-e2e-test resourceVersion: "25583" uid: f1c4a1b9-814c-41c4-8fa0-50fe8ee596db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4831c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe209cdf8202c namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: 6a2b44c4-cfd3-413b-ad65-c4f05ab39a8e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25589" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a02089086 namespace: kserve-ci-e2e-test resourceVersion: "25620" uid: dbc0347d-6c1e-40f8-8acb-de3d60b5e3b6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a0375a959 namespace: kserve-ci-e2e-test resourceVersion: "25622" uid: 8a57c622-15b0-4227-955f-bfb8f55338e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a06963191 namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: 835a2b5c-2248-41f9-a75f-f05d22e55993 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a070d7c82 namespace: kserve-ci-e2e-test resourceVersion: "25624" uid: 54ab50f3-560e-4052-b2e0-f9261d9b4edd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a071a4d14 namespace: kserve-ci-e2e-test resourceVersion: "25625" uid: 47f2a556-c530-47c9-a0c3-cd5bce3edde8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a0b22dec2 namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: c38a6a99-9ac2-4284-a247-d5fce6354e0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:48:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:30Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a0ba246e2 namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: dcd9b6d2-51fd-47da-9c1d-09e99e1a72ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:49:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:07Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe20a3c291c11 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: fb7b79fa-2dc9-49f2-a557-d1d97b37133c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe21c734698ac namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 25ab661e-e32e-4e54-a72f-2f4c246e971e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe21c7346fb07 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: fc95690b-8f72-4abc-96ef-2e8d487f0a6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4831c-predictor-5944766686-tvxdv namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 5d5562c9-3add-4214-8263-1d68f69f9b48 kind: Event lastTimestamp: "2026-07-07T02:49:52Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:52Z" name: error-404-isvc-4831c-predictor-5944766686-tvxdv.18bfe21d1857038b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 36c94fb0-7fdd-4492-bfdb-3162013db9bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4831c-predictor-5944766686 namespace: kserve-ci-e2e-test resourceVersion: "25569" uid: 2a01d6c9-8693-4601-9f60-289a76014d41 kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: 'Created pod: error-404-isvc-4831c-predictor-5944766686-tvxdv' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:29Z" name: error-404-isvc-4831c-predictor-5944766686.18bfe209c36ec9aa namespace: kserve-ci-e2e-test resourceVersion: "25580" uid: daeebf70-cd99-46e1-b90d-16b6285f1580 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: 76784dc3-ddb5-4bdf-a82e-6a3b13574cdc kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Scaled up replica set error-404-isvc-4831c-predictor-5944766686 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:28Z" name: error-404-isvc-4831c-predictor.18bfe209c18ab6ed namespace: kserve-ci-e2e-test resourceVersion: "25571" uid: 1f3be783-5679-4fe7-9aac-2d983316b437 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25568" uid: db577af3-79c1-4ed8-8098-0a4b5d97d669 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:59Z" name: error-404-isvc-4831c-predictor.18bfe20d3ff8839a namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 6d0ee337-2bf5-4f77-9edc-52c398eb7f43 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:48:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25568" uid: db577af3-79c1-4ed8-8098-0a4b5d97d669 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:59Z" name: error-404-isvc-4831c-predictor.18bfe20d405d7272 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: 8345dd01-b0d5-4ca4-8a1c-0baed547523c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: db577af3-79c1-4ed8-8098-0a4b5d97d669 kind: Event lastTimestamp: "2026-07-07T02:49:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:29Z" name: error-404-isvc-4831c-predictor.18bfe2143fe1d220 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 2f2925e5-244e-48f8-9416-be5f7360934d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: db577af3-79c1-4ed8-8098-0a4b5d97d669 kind: Event lastTimestamp: "2026-07-07T02:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:29Z" name: error-404-isvc-4831c-predictor.18bfe21440404943 namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: b4951c68-6f76-43e8-b775-f3c6866e7bce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "25537" uid: 8290a30e-f3c3-471f-aff1-13a73a5be9a0 kind: Event lastTimestamp: "2026-07-07T02:49:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:17Z" name: error-404-isvc-4831c.18bfe209c18e37fc namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: fc8e9777-a9dc-462c-8918-d18e68eac34e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "25537" uid: 8290a30e-f3c3-471f-aff1-13a73a5be9a0 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-4831c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4831c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:48:29Z" name: error-404-isvc-4831c.18bfe209c2021a2e namespace: kserve-ci-e2e-test resourceVersion: "25575" uid: 230c3910-25b8-4119-ba00-becf29605493 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "25537" uid: 8290a30e-f3c3-471f-aff1-13a73a5be9a0 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4831c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:48:29Z" name: error-404-isvc-4831c.18bfe209c20259bb namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: 79438e75-e36c-4600-8348-d0316f48ffb4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 8290a30e-f3c3-471f-aff1-13a73a5be9a0 kind: Event lastTimestamp: "2026-07-07T02:49:17Z" message: InferenceService [error-404-isvc-4831c] is Ready metadata: creationTimestamp: "2026-07-07T02:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:17Z" name: error-404-isvc-4831c.18bfe214f87bc81b namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 33008dd3-2cdb-4b04-bb57-aba012519f95 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:38:33.708553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-524cd-predictor-58fd54b564-2w825 to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f27e72232 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 9b51c47c-e59c-40c7-9d21-9a37c04b1890 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-524cd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f34938dae namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 53ed66ce-15dc-4705-a5b3-ef342fdc2c59 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f65cc0a46 namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: e931c3e7-20cc-4fc1-8eb5-cdb93c66f316 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f671e0f03 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 29b9cde3-5730-4b6c-a3ef-c6d48ad90a09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f69c9ae78 namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 438bb844-c4ab-452b-b1d9-40b1d87f347d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f6a47c316 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 0405555b-bccd-4b5d-befa-51b7ff60909e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f6a53fb30 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 3314cfce-f59e-4a5b-9b06-88dad1fcca6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f6d48de5f namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 197d8f9f-9cb0-4c30-a859-f31dc29cca82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f6dc26317 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 5d150a06-624a-48e5-9888-bd990c71b497 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:39:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:11Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe17f99ccb125 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 3c9a5fbd-6e07-41bd-beba-ed2ee88baadb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe18f7c14497d namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 091c7788-80d3-4fa4-ad47-9e7412e1667b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe18f7c14fffe namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 71dbc1ac-86f9-4107-9603-b2ad59494003 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-524cd-predictor-58fd54b564-2w825 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 86fa344a-3fb0-4d92-8c46-d3d4d24801a5 kind: Event lastTimestamp: "2026-07-07T02:39:46Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:46Z" name: error-404-isvc-524cd-predictor-58fd54b564-2w825.18bfe19021f2fff5 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: f1f1edef-a78e-42b1-afc2-f8cde3cdf7ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-524cd-predictor-58fd54b564 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 47ee41b8-ae58-4951-97d9-6d85f8c00e28 kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'Created pod: error-404-isvc-524cd-predictor-58fd54b564-2w825' metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-524cd-predictor-58fd54b564.18bfe17f275cb24d namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 8e96eb66-3bbf-443c-828a-353d5d94db0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 6794dd9d-fd9f-4be6-b2e5-05814966203e kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Scaled up replica set error-404-isvc-524cd-predictor-58fd54b564 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-524cd-predictor.18bfe17f257cd8e1 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: a9d411b9-30e9-45d7-bfea-fb94d49f00f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: dfcbcc83-4074-498b-b666-c85d4587e7f6 kind: Event lastTimestamp: "2026-07-07T02:38:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:48Z" name: error-404-isvc-524cd-predictor.18bfe182a3fe1de4 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 44165987-b2c9-42bc-ad68-12b0afc0235d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: dfcbcc83-4074-498b-b666-c85d4587e7f6 kind: Event lastTimestamp: "2026-07-07T02:38:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:48Z" name: error-404-isvc-524cd-predictor.18bfe182a467c50b namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 19a91590-b8a7-4098-b98b-ba0325e8e90b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: dfcbcc83-4074-498b-b666-c85d4587e7f6 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:33Z" name: error-404-isvc-524cd-predictor.18bfe186234c24c2 namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: c4eacfc5-a930-4424-be8c-8415492b6455 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: dfcbcc83-4074-498b-b666-c85d4587e7f6 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:33Z" name: error-404-isvc-524cd-predictor.18bfe18623b4d863 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: a296ec45-35db-4306-b271-6792d8b066f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-524cd namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 0630df85-a7e5-45a6-b901-c8ea1d2b036e kind: Event lastTimestamp: "2026-07-07T02:39:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:21Z" name: error-404-isvc-524cd.18bfe17f2585dd65 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: cd1541f3-b9f4-4951-b9cb-b15a41da2c55 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-524cd namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 0630df85-a7e5-45a6-b901-c8ea1d2b036e kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-524cd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-524cd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-524cd.18bfe17f25e8b59d namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 751ad73c-4ce6-492f-9c23-6c48d3a37d55 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-524cd namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 0630df85-a7e5-45a6-b901-c8ea1d2b036e kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-524cd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-524cd.18bfe17f25e8f8bb namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: 0a99633b-0ae8-4aa7-82e2-7037cc2feb81 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-524cd namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 0630df85-a7e5-45a6-b901-c8ea1d2b036e kind: Event lastTimestamp: "2026-07-07T02:39:21Z" message: InferenceService [error-404-isvc-524cd] is Ready metadata: creationTimestamp: "2026-07-07T02:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:21Z" name: error-404-isvc-524cd.18bfe18a54abede8 namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: e005af7a-0c6e-40fc-9f38-ff152016c07f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:39:43.948786Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7c238-predictor-c4bd4d474-qb4ww to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18f828bfa57 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 04984a1a-6329-4e8d-9401-f2f204d532a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18f9c42304c namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: 9ac35e15-37b9-434b-a19d-9330d9daa569 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18f9d8e536e namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 67a7e719-3c96-4aa4-a744-085325a72f12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18fa0247bf7 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 28ccc1a7-7b7a-4bdf-9003-0ac5d035c704 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18fa09c9cf1 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 8c9c13bd-e69c-4aba-aafa-c3013b0c503a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18fa0a6ab21 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 273fe5d4-93e4-4f20-95b6-6139e3c601d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18fa5173d14 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 0c8ebe36-df4a-48cf-8b39-9e5cc015cac4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18fa5906fde namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 81c8ac1b-4e13-4ed1-b66d-1ae363a7d45c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe18fbc208520 namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: 94584319-fa73-49e4-bc17-a74c40002f33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe210b469e9d1 namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: 0b5a06ab-6600-4398-8c5a-178e309bd27e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe210b46b1a31 namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: e5ec4244-d472-4db1-81f3-917460ea52cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c1875431-a285-4261-9613-9843f0ece766 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww.18bfe211308d745e namespace: kserve-ci-e2e-test resourceVersion: "26021" uid: b5b694ae-e9c4-477e-be78-222e8ed93e1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7c238-predictor-c4bd4d474 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: eaab7c1e-5014-4cb0-88da-5d0b8b91fa6a kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'Created pod: error-404-isvc-7c238-predictor-c4bd4d474-qb4ww' metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-7c238-predictor-c4bd4d474.18bfe18f8213797e namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 587560ce-1a36-4ef3-9065-b70a2392485e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: b64f3b7b-c92b-4060-9673-40ffc49ddf6f kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Scaled up replica set error-404-isvc-7c238-predictor-c4bd4d474 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-7c238-predictor.18bfe18f8069366c namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d64c1ae-9e68-47be-af38-07d49c4d7479 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 1f310674-c2c4-4ba4-9e69-4c9e2efc12ba kind: Event lastTimestamp: "2026-07-07T02:40:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:13Z" name: error-404-isvc-7c238-predictor.18bfe192feda66eb namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: 6b2da164-0e6c-43e7-ad0f-eda64dc10a9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 1f310674-c2c4-4ba4-9e69-4c9e2efc12ba kind: Event lastTimestamp: "2026-07-07T02:40:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:13Z" name: error-404-isvc-7c238-predictor.18bfe192ff409ebf namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 0bd18e8a-d4a9-4de0-8cb4-d4a3df486402 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 1f310674-c2c4-4ba4-9e69-4c9e2efc12ba kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:43Z" name: error-404-isvc-7c238-predictor.18bfe199fc93970c namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: c04dfbdc-7753-48b9-b449-cbc79934c02d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 1f310674-c2c4-4ba4-9e69-4c9e2efc12ba kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:43Z" name: error-404-isvc-7c238-predictor.18bfe199fcfc2fa4 namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 55f984c1-f387-4531-b979-8dcd1ce064c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: e9bdae7f-af30-48ac-951c-540482fd0080 kind: Event lastTimestamp: "2026-07-07T02:40:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:31Z" name: error-404-isvc-7c238.18bfe18f806b1f94 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 709a5f03-6754-4e94-b6da-4af30d176044 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: e9bdae7f-af30-48ac-951c-540482fd0080 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-7c238": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c238": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-7c238.18bfe18f80f77db5 namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: 684c7516-c0a2-43ab-b14b-1ea958289e1b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: e9bdae7f-af30-48ac-951c-540482fd0080 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c238": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:43Z" name: error-404-isvc-7c238.18bfe18f80f7c98b namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 74fb55dd-6ea7-4f22-a0a7-9a180335b0cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: e9bdae7f-af30-48ac-951c-540482fd0080 kind: Event lastTimestamp: "2026-07-07T02:40:31Z" message: InferenceService [error-404-isvc-7c238] is Ready metadata: creationTimestamp: "2026-07-07T02:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:31Z" name: error-404-isvc-7c238.18bfe19a785db41a namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 1ab4f1f2-118a-493d-9b69-04209987a4da reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:29:18.701167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:29:18Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fdeee28c67 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 6ce7cd8b-c9b6-49e3-b76b-dd02ad849b5e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe08ab68d7 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 956615d7-6eb1-44a9-a87c-dc58614108e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe0a3ff673 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 460d2408-1a5a-495c-b5ae-78178eaac3a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe0cd6e6ed namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: c6f87760-5abe-4fbf-a06b-65b784706a21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe0d40c338 namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 3ab0e35a-434f-4ea1-a79e-bd885ad891ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe0d4b1351 namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 0b24201b-b75e-4dcf-bf89-48e09289627e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe0fe9941f namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: fc00313b-4827-446b-a6fd-fa6384d54240 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe105a9a53 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: b9cde1a8-3539-44b8-99c3-d5bf9ec4e64c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:38:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:35Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe0fe125d5083 namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: aaa78c18-c746-4188-af4a-ea33c64ccd6a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe17f20cbe9aa namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: 7bfd11ae-819a-4ca8-9a21-c46255eb0d97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe17f20d12154 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 517c640d-366b-4d7b-9d49-25613a082df8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 44e3421f-0f5c-4c2d-8993-c1d1556b7f1d kind: Event lastTimestamp: "2026-07-07T02:38:35Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:35Z" name: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5.18bfe17f86cce4e0 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: e98e861e-a31a-4f0f-9154-bf295477b6aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-821ea-predictor-68c7476dc4 namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: c481dba1-bf96-4242-8190-3982621d42b8 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: 'Created pod: error-404-isvc-821ea-predictor-68c7476dc4-gs9p5' metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:18Z" name: error-404-isvc-821ea-predictor-68c7476dc4.18bfe0fdee863d32 namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 2ae561f2-00ea-4925-a62d-522a14e03674 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 8b6a17fd-b882-46b3-8945-3c404c8c916e kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Scaled up replica set error-404-isvc-821ea-predictor-68c7476dc4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:18Z" name: error-404-isvc-821ea-predictor.18bfe0fdec7763d5 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: da052bca-4d59-40ac-a9f2-17fb6174b103 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:29:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: e258c403-65af-4402-ab6c-7ccfab602fa2 kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:48Z" name: error-404-isvc-821ea-predictor.18bfe1016add7bed namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: dbaced6f-b0d4-43c0-ac70-7dfcb0f56fc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:29:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: e258c403-65af-4402-ab6c-7ccfab602fa2 kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:48Z" name: error-404-isvc-821ea-predictor.18bfe1016b951319 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 1b27ff74-1af5-47aa-a8c5-267bf43a39a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:30:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: e258c403-65af-4402-ab6c-7ccfab602fa2 kind: Event lastTimestamp: "2026-07-07T02:30:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:33Z" name: error-404-isvc-821ea-predictor.18bfe1086937244d namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: cc1a45b0-e610-49e1-b57d-577e5a4f02d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:30:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: e258c403-65af-4402-ab6c-7ccfab602fa2 kind: Event lastTimestamp: "2026-07-07T02:30:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:33Z" name: error-404-isvc-821ea-predictor.18bfe1086982c7d3 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: d51479eb-0461-4e26-8e56-9810727494bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-821ea namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 4436d089-ab14-49c5-b683-dc2ea92625aa kind: Event lastTimestamp: "2026-07-07T02:30:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:30:05Z" name: error-404-isvc-821ea.18bfe0fdec6b276b namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 20c2c40f-cb1f-4402-a7bc-c6ac2cbad130 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-821ea namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 4436d089-ab14-49c5-b683-dc2ea92625aa kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-821ea": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-821ea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:18Z" name: error-404-isvc-821ea.18bfe0fded3af840 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: f3a3cfd9-19af-40e4-a1d1-135eef103485 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-821ea namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 4436d089-ab14-49c5-b683-dc2ea92625aa kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-821ea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:18Z" name: error-404-isvc-821ea.18bfe0fded3b4646 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 6bb103e9-182e-4429-af4f-60f459543b45 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-821ea namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 4436d089-ab14-49c5-b683-dc2ea92625aa kind: Event lastTimestamp: "2026-07-07T02:30:05Z" message: InferenceService [error-404-isvc-821ea] is Ready metadata: creationTimestamp: "2026-07-07T02:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:30:05Z" name: error-404-isvc-821ea.18bfe108ce595a89 namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 8365ad4d-f0ee-49e5-9d8a-86d875ec046b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:49:49.408735Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8f974-predictor-854b47f8fb-ljkwj to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21c7ac5dde0 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 4c10ba64-40de-4951-8fb1-5b6f28e0af30 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8f974-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21c8944e8d8 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: b0040184-7721-4425-a1d9-a845c535370e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cb8973dee namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: a863757a-73a7-473d-86c7-7921cbacf68c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cba1ec26b namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: 2c6d758a-4d47-46cc-bf5a-4e87ad897a16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cbdf95c7d namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: cf1788f7-af31-4f2f-acfd-2cf95b4641a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cbe77665c namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 701aeafb-19c1-4f75-a539-6936c98e764e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cbe828be9 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: dd4a56de-10fe-462b-9d76-2aad7741b3dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cc1d420e9 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: fff2b00a-cb59-4429-853d-6046d6dd65e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cc267bb96 namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: ec199bc3-8ced-46c7-92a3-f53ec688af04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:50:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:50:27Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe21cf16ef052 namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 5c869b95-2109-4223-9bac-e804d8e9f397 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:59:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:04Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe29da3618d7f namespace: kserve-ci-e2e-test resourceVersion: "30668" uid: ba770c1e-d10c-4be3-99e8-0635a971773a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:59:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:04Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe29da36267c3 namespace: kserve-ci-e2e-test resourceVersion: "30669" uid: 6ddf5fb2-43b0-4863-87a1-d6b552a50d7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:59:07Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:07Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe29e65c67e57 namespace: kserve-ci-e2e-test resourceVersion: "30697" uid: 7c12a066-55e2-422b-8b19-0a7f02e4a8a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 7ce7b8ee-a3c6-4422-b8c0-cb655b0e8a44 kind: Event lastTimestamp: "2026-07-07T02:59:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: i/o timeout' metadata: creationTimestamp: "2026-07-07T02:59:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:08Z" name: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj.18bfe29ea1af557f namespace: kserve-ci-e2e-test resourceVersion: "30708" uid: 27d92f84-5cd8-4267-9daf-c3701c2c9021 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8f974-predictor-854b47f8fb namespace: kserve-ci-e2e-test resourceVersion: "26580" uid: 5471c58b-72f2-48d0-bd47-b997cfbe6dfd kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'Created pod: error-404-isvc-8f974-predictor-854b47f8fb-ljkwj' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-8f974-predictor-854b47f8fb.18bfe21c7a52936f namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 06f07709-b4a4-416b-a17f-92135c7dd088 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: ecd4170e-3f8d-4abc-a347-f1739c911954 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Scaled up replica set error-404-isvc-8f974-predictor-854b47f8fb from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-8f974-predictor.18bfe21c787f0fb8 namespace: kserve-ci-e2e-test resourceVersion: "26581" uid: 4994e3fe-69c1-4680-a85c-53eedd0e48a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: 77c1b73c-2181-43d1-9ca4-97036cd143d0 kind: Event lastTimestamp: "2026-07-07T02:50:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:04Z" name: error-404-isvc-8f974-predictor.18bfe21ff6fa03c8 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: c352af80-8673-4522-93a7-4e301e4b06c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: 77c1b73c-2181-43d1-9ca4-97036cd143d0 kind: Event lastTimestamp: "2026-07-07T02:50:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:04Z" name: error-404-isvc-8f974-predictor.18bfe21ff760ff04 namespace: kserve-ci-e2e-test resourceVersion: "26792" uid: f1715b8a-ad2f-4e7d-bf7d-abc7a45f8fc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:50:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: 77c1b73c-2181-43d1-9ca4-97036cd143d0 kind: Event lastTimestamp: "2026-07-07T02:51:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:04Z" name: error-404-isvc-8f974-predictor.18bfe22376153e5c namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 05e12a89-c58f-487a-bbe3-40ba4204a4f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:50:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: 77c1b73c-2181-43d1-9ca4-97036cd143d0 kind: Event lastTimestamp: "2026-07-07T02:51:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:04Z" name: error-404-isvc-8f974-predictor.18bfe223767f7dae namespace: kserve-ci-e2e-test resourceVersion: "27314" uid: 25e8fa2b-6700-4617-a373-ae760b1d2eaa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 93674155-ffc4-4d74-9401-1a1cfa2e7ab6 kind: Event lastTimestamp: "2026-07-07T02:50:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:50:37Z" name: error-404-isvc-8f974.18bfe21c788426ef namespace: kserve-ci-e2e-test resourceVersion: "27074" uid: 45791dbd-1e78-4bf2-84eb-830a1998d76f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 93674155-ffc4-4d74-9401-1a1cfa2e7ab6 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-8f974": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8f974": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-8f974.18bfe21c791a4f10 namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: b38f2ec0-72b4-4eef-916a-8e1405848877 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 93674155-ffc4-4d74-9401-1a1cfa2e7ab6 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8f974": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:49Z" name: error-404-isvc-8f974.18bfe21c791aa67c namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: 888b478b-26ed-4165-8e88-0db35c63f61a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27072" uid: 93674155-ffc4-4d74-9401-1a1cfa2e7ab6 kind: Event lastTimestamp: "2026-07-07T02:50:37Z" message: InferenceService [error-404-isvc-8f974] is Ready metadata: creationTimestamp: "2026-07-07T02:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:50:37Z" name: error-404-isvc-8f974.18bfe227ac715297 namespace: kserve-ci-e2e-test resourceVersion: "27073" uid: 125ee08f-1360-4970-9ad1-a21084dc15f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:48:58.958764Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2091-predictor-565f465b48-pprvw to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210bbb84071 namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: aa098936-b597-4ba9-94ef-bafdeed192f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b2091-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210c6ec1e72 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: c16b5bdc-f792-43aa-9624-ff817cf06776 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210f9c27031 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 74cd8194-dc5b-49e0-bdad-db4d70a00bd3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210fb64a0f9 namespace: kserve-ci-e2e-test resourceVersion: "26007" uid: 64bf6cf8-4d0a-4b8f-85be-75245c824d2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210fe94e19b namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 35f41b9f-612a-4009-9db8-aed238106aad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210ff1121b0 namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: 20e7c36f-b4ca-47e6-9280-e20fb5cadcec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe210ff1bdec7 namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 76c96ebb-3a14-4e33-ac62-4d50a92f7be1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe211023de642 namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: eafa5782-f43a-4081-aa47-146c0a8bed6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:00Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe21102c0c4d7 namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: cf1c1b63-7556-48e0-907b-fdd0450436ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T02:49:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:37Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe2114033dfaa namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: 32252182-9b3d-4e87-bd48-00e2889f8c96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T03:06:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T03:06:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T03:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:18Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe302b65a52ba namespace: kserve-ci-e2e-test resourceVersion: "33668" uid: 4d0cb7c0-a032-4530-9720-bae20a8d26e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T03:06:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2091-predictor-565f465b48-pprvw namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 44124dcf-f193-492c-a991-7f6072bd3fb0 kind: Event lastTimestamp: "2026-07-07T03:06:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T03:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:18Z" name: error-404-isvc-b2091-predictor-565f465b48-pprvw.18bfe302b65e8130 namespace: kserve-ci-e2e-test resourceVersion: "33669" uid: 0b702b35-c13b-4b59-8d61-a1078fdd0233 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2091-predictor-565f465b48 namespace: kserve-ci-e2e-test resourceVersion: "25950" uid: c709e8c3-9486-45bd-8cfd-d0f97e3f4a76 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: 'Created pod: error-404-isvc-b2091-predictor-565f465b48-pprvw' metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-b2091-predictor-565f465b48.18bfe210bb538ce6 namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: e927474c-b1e8-473e-b3f3-814852114f64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: d73ba67a-ad2f-488e-8f41-a46585deeb12 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Scaled up replica set error-404-isvc-b2091-predictor-565f465b48 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-b2091-predictor.18bfe210b8cc4e8a namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: ee90f10a-6270-4f8a-8e59-ae2b2e9476d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 64ef2d17-1cd8-40c7-a13a-eac94fff7a8a kind: Event lastTimestamp: "2026-07-07T02:49:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:13Z" name: error-404-isvc-b2091-predictor.18bfe214374c0781 namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: a1ac020f-2920-44b2-a804-780140802dce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 64ef2d17-1cd8-40c7-a13a-eac94fff7a8a kind: Event lastTimestamp: "2026-07-07T02:49:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:13Z" name: error-404-isvc-b2091-predictor.18bfe21437b315c2 namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: be8400d5-1d43-410f-adf2-694073ecc645 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:49:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 64ef2d17-1cd8-40c7-a13a-eac94fff7a8a kind: Event lastTimestamp: "2026-07-07T02:50:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:13Z" name: error-404-isvc-b2091-predictor.18bfe217b65d1170 namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 85374418-81cb-4891-990f-670e09a2f171 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:49:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 64ef2d17-1cd8-40c7-a13a-eac94fff7a8a kind: Event lastTimestamp: "2026-07-07T02:50:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:13Z" name: error-404-isvc-b2091-predictor.18bfe217b6a2df6a namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: d9fcfa78-2220-4479-aca5-c999e42c5357 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2091 namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: eac8db45-cbac-4a34-ad62-d713051b9c49 kind: Event lastTimestamp: "2026-07-07T02:49:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:47Z" name: error-404-isvc-b2091.18bfe210b8c881ec namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: f38b96a2-cca4-4529-b3dc-2c22bfb305a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2091 namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: eac8db45-cbac-4a34-ad62-d713051b9c49 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-b2091": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2091": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-b2091.18bfe210b94adadb namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 3ca3ffbd-1700-4771-b432-d556edda5e83 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2091 namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: eac8db45-cbac-4a34-ad62-d713051b9c49 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2091": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:48:58Z" name: error-404-isvc-b2091.18bfe210b94b3fa9 namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 30bf96ba-d180-446c-9348-7d9028836c35 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: eac8db45-cbac-4a34-ad62-d713051b9c49 kind: Event lastTimestamp: "2026-07-07T02:49:47Z" message: InferenceService [error-404-isvc-b2091] is Ready metadata: creationTimestamp: "2026-07-07T02:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:47Z" name: error-404-isvc-b2091.18bfe21bfb55616e namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 341d27bb-1470-4f04-8e25-575e2100d189 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:38:03.685733Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1782a674ffc namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 05463a5d-d52e-42bf-bd5e-875bf421dbc5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-cec47-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe178359be6ba namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 4581417e-f419-4430-8147-a19c5cfd3a76 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20070" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe178683b8822 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: f0c41040-d4b6-4ff1-84ac-a74dc0867542 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe17869942bff namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: fa5d844a-d318-4b02-b340-6ea426485f26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1786c601971 namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 75a340d5-72f1-4b55-8db8-2c740f7dafb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1786cca303e namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 2b3c940a-276f-42fd-a381-d8f29412eeb4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1786cd3d4ca namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 3744462c-ac5d-4b7f-ad4a-283108a0ecb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1786f76784e namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: ac3f9538-2e95-430c-994c-299086901842 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1786fe3962c namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 752ecee6-9f2b-4071-a375-021e82d64fe0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:38:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:42Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe178d1942424 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: f8329e2c-d589-40d7-882c-28487009923e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:13Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1887f54b5a8 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 9d04c531-1cb2-45de-8dd6-9f924cde5056 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: a6310814-479f-42c5-9eba-995181462a21 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:13Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd.18bfe1887f5529e7 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: ed5ac4e7-6b6b-4af7-86ae-7e3cf4d5bbc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cec47-predictor-6dccdfd5c6 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: d34ad0b4-b943-4d0e-a0e5-f7865119c579 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'Created pod: error-404-isvc-cec47-predictor-6dccdfd5c6-ldwhd' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-cec47-predictor-6dccdfd5c6.18bfe17829f968cd namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: c5212023-d9ea-4f0e-b4a4-d7307b34b8c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: d7a7cd4b-d0b8-42a9-bc68-ccc24e19119b kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Scaled up replica set error-404-isvc-cec47-predictor-6dccdfd5c6 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-cec47-predictor.18bfe178286d6366 namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bc4b6a29-d6dd-4a5d-bc71-1b7e91714779 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:38:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: b58537b6-1dae-4c8e-b130-6337d6ae2848 kind: Event lastTimestamp: "2026-07-07T02:38:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:48Z" name: error-404-isvc-cec47-predictor.18bfe17ba6f8042b namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 29f2342f-38cd-4363-96c4-dc5408ac2caf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:38:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: b58537b6-1dae-4c8e-b130-6337d6ae2848 kind: Event lastTimestamp: "2026-07-07T02:38:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:48Z" name: error-404-isvc-cec47-predictor.18bfe17ba7462413 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: d6632417-4ed8-4632-bf69-801f806c66f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: b58537b6-1dae-4c8e-b130-6337d6ae2848 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: error-404-isvc-cec47-predictor.18bfe18623871d9d namespace: kserve-ci-e2e-test resourceVersion: "20778" uid: 29caacb5-ae7f-4a4a-a8bf-6c81b80f6e5f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: b58537b6-1dae-4c8e-b130-6337d6ae2848 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: error-404-isvc-cec47-predictor.18bfe18623e0355c namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: c1fbc163-de4a-4184-b393-014fb1e02eb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 5f9cdeb4-c7de-4e9f-a7db-3f7b9d8cd71f kind: Event lastTimestamp: "2026-07-07T02:38:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:52Z" name: error-404-isvc-cec47.18bfe1782873ed79 namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: 5b286cee-11c1-4ac4-bdf0-b1e57b2e7dc3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 5f9cdeb4-c7de-4e9f-a7db-3f7b9d8cd71f kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-cec47": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cec47": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-cec47.18bfe17828dbfb0c namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: ec64cf85-41c3-4540-b1bf-4cc0e793e097 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 5f9cdeb4-c7de-4e9f-a7db-3f7b9d8cd71f kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cec47": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:03Z" name: error-404-isvc-cec47.18bfe17828dc4949 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: df5bec0b-89bd-41f3-b4c2-89b713d28003 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 5f9cdeb4-c7de-4e9f-a7db-3f7b9d8cd71f kind: Event lastTimestamp: "2026-07-07T02:38:52Z" message: InferenceService [error-404-isvc-cec47] is Ready metadata: creationTimestamp: "2026-07-07T02:38:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:52Z" name: error-404-isvc-cec47.18bfe1838c9c7ffe namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 674f8f4f-db2e-473b-a23d-40dc1fbd8630 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:39:14.027386Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e4a8c-predictor-669cdb968b-nfffg to ip-10-0-142-230.ec2.internal metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:39:14Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe1888b17be21 namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 69dbee6b-6b0f-4d6c-992b-f8bebe1dcdc7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e4a8c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:14Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe1889777ae6c namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: e94a8640-7158-4c5e-a543-f395b041dfaf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188c8b7b469 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: a28a52fc-f117-4f8b-bd38-1705c52c011a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188ca0e433c namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 17f3c0ba-a0d4-4d87-8b91-c5ebdb0ad7d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188ccff8e00 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: c187ee49-9895-445e-a89f-a35400ae1d52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188cdbac414 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 7c91cb79-12d8-4d88-b33e-a6a5edf27dca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188cdc54795 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 73ab3ee5-7925-4bdf-a3fd-7f361250bddf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188d0928491 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: fd02f8e0-73ba-4bea-9ded-827a32475a68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe188d1079fe6 namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 51b6eaec-0f90-489c-b8cb-417871da1a6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:39:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:39:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:52Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe1892ff4f550 namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 43a6b258-2658-418c-a555-93fa3b176d58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:28Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe209c0fbe0ba namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: 88b0edeb-55c5-4954-90ee-4ff41e453c50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 00124b43-38ed-4308-a2f1-e997b3a23856 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:28Z" name: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg.18bfe209c10210eb namespace: kserve-ci-e2e-test resourceVersion: "25570" uid: 48042d33-fbdd-497b-8466-3076c5b640b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-230.ec2.internal source: component: kubelet host: ip-10-0-142-230.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e4a8c-predictor-669cdb968b namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: 44338a3c-9e71-4b04-a11e-ded1d1b97d38 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: 'Created pod: error-404-isvc-e4a8c-predictor-669cdb968b-nfffg' metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:14Z" name: error-404-isvc-e4a8c-predictor-669cdb968b.18bfe1888ab853ab namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: f2bed475-4be0-45fd-ae04-636738f43f63 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: d272a793-7af6-49b8-a70b-b40198781ecf kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Scaled up replica set error-404-isvc-e4a8c-predictor-669cdb968b from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:13Z" name: error-404-isvc-e4a8c-predictor.18bfe188892063d8 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: 10212e8c-3bd3-4a97-ba54-a101135d983d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: d2de6de5-c58d-4171-b48e-dad3b9ccb528 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-e4a8c-predictor.18bfe18c07a9b826 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 1d32a53f-891e-484e-910a-4dbfc0bf098e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: d2de6de5-c58d-4171-b48e-dad3b9ccb528 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:44Z" name: error-404-isvc-e4a8c-predictor.18bfe18c08113575 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 70337bf8-9911-4a3d-88a2-8acc60160eb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: d2de6de5-c58d-4171-b48e-dad3b9ccb528 kind: Event lastTimestamp: "2026-07-07T02:40:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:14Z" name: error-404-isvc-e4a8c-predictor.18bfe1930684f83a namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: cabb863f-5a34-4111-80d2-fcf034250a80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: d2de6de5-c58d-4171-b48e-dad3b9ccb528 kind: Event lastTimestamp: "2026-07-07T02:40:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:14Z" name: error-404-isvc-e4a8c-predictor.18bfe19306c943ef namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: b9e82151-3574-4c8a-a62f-174fa7a7406b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 10f3b25b-7855-480e-8c5d-2a61db55dab3 kind: Event lastTimestamp: "2026-07-07T02:40:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:02Z" name: error-404-isvc-e4a8c.18bfe1888927433d namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 7d17f369-ffee-4f98-92e2-01efae38fe35 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 10f3b25b-7855-480e-8c5d-2a61db55dab3 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-e4a8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:14Z" name: error-404-isvc-e4a8c.18bfe18889b7eb41 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 318132e5-43e1-4fb1-8185-49b2c11da5fe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 10f3b25b-7855-480e-8c5d-2a61db55dab3 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:14Z" name: error-404-isvc-e4a8c.18bfe18889b834ed namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 05a54b90-c5ed-489b-bd83-9c755e1f113c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 10f3b25b-7855-480e-8c5d-2a61db55dab3 kind: Event lastTimestamp: "2026-07-07T02:40:02Z" message: InferenceService [error-404-isvc-e4a8c] is Ready metadata: creationTimestamp: "2026-07-07T02:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:02Z" name: error-404-isvc-e4a8c.18bfe193eb6b51d0 namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 68d8a04c-e253-41ca-a9d2-637d46bd7158 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:26:58.462165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0dd47fd6bad namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: e72efd7f-47da-4ac8-9af8-1e5a8c4f199b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:59Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0dd513d375d namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 131b732b-a376-418b-9da9-9a60611cf63e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:00Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:27:00Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0ddc075ffaf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 8d431805-daeb-434d-82b6-bf5bd48e305e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:f221c928a0bb1ba5e4c45ae19b5fe18d4abac3f5b57f42edecf256c4242d4d40" already present on machine metadata: creationTimestamp: "2026-07-07T02:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:00Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0ddc2ab31d5 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: c0f3f4fd-5ba2-4636-8644-0e25aac1aad0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T02:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:00Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0ddc8538409 namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 4ea0162b-912e-4acc-80c5-3d062e1ea494 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T02:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:00Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0ddc9598833 namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 806d4158-b944-4d82-b48d-b70aaa35bb8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1702" metadata: creationTimestamp: "2026-07-07T02:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:04Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0dec53abd47 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 9c244b21-711a-4ab3-8f15-e20282c92830 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1702" in 10.871s (10.871s including waiting). Image size: 1555967903 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e14d37b71e namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: b7ca4f4a-fed2-421a-9c0c-c3966be08ca8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e150ca4d15 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 347d884a-a6c9-443c-b3be-ec1df6892df1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e15151b056 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: bd028da5-8e69-4e3a-8a3a-d62dcb9a3446 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e1515f3c46 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 6d487efd-852d-446c-b0e6-171ca611498a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.133s (4.133s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:19Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e247c4643d namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: d1bcd17c-0daa-4a1b-aec5-3ed5b1fc0997 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:20Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e24e508761 namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 1fe0530c-e1a8-43a8-b6f8-2546878109cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:27:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:20Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e24ee8168c namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 8f444819-9848-4d05-b7f4-16f0f8361d3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T02:27:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:29:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:20Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0e2c2c5739b namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: c474f42a-b4eb-4be2-97b5-6b5d107c8a5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0fde93c4a58 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 70bc53c0-40bf-44f3-90d5-46fa868c42c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0fde93d5e18 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 94413420-dbe0-4949-9024-f5660d3a6beb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: e17e9087-204f-4750-9bf3-762232f66d66 kind: Event lastTimestamp: "2026-07-07T02:29:23Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:23Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx.18bfe0feef317563 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: ed8cfad2-73c1-4fcb-ab55-830b961a5752 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-658dd9c69f namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 4048e2eb-43e8-45b7-a05e-2519329f9399 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-658dd9c69f-csnfx' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-1-predictor-658dd9c69f.18bfe0dd4784f434 namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 046b6c84-a5aa-4c81-b17a-e44f6bb2f202 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: 9727c7af-6adf-4b61-922f-977eb7dbd41d kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-658dd9c69f from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-1-predictor.18bfe0dd44a2865a namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 2490dafc-3162-4219-a2bc-192e8c08910a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 52d58dab-7e10-4ac7-aee2-99f57b452a82 kind: Event lastTimestamp: "2026-07-07T02:27:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:43Z" name: isvc-sklearn-graph-1-predictor.18bfe0e0c37cf373 namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: facf1394-9923-4f6a-9614-f4986a2e12ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 52d58dab-7e10-4ac7-aee2-99f57b452a82 kind: Event lastTimestamp: "2026-07-07T02:27:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:43Z" name: isvc-sklearn-graph-1-predictor.18bfe0e0c3f9b884 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: bf97d95d-7dbf-4e9d-832b-1dde86138d60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: 52d58dab-7e10-4ac7-aee2-99f57b452a82 kind: Event lastTimestamp: "2026-07-07T02:28:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:58Z" name: isvc-sklearn-graph-1-predictor.18bfe0eb4121c1c2 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: a9b02137-5ad4-4f1c-a9f8-02ae2499a7bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: 52d58dab-7e10-4ac7-aee2-99f57b452a82 kind: Event lastTimestamp: "2026-07-07T02:28:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:58Z" name: isvc-sklearn-graph-1-predictor.18bfe0eb41d2e04c namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: dbbc5fe6-5638-4f01-8ec5-54051f320b9c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: da6fad0a-4757-42ab-9f73-252676507953 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:40Z" name: isvc-sklearn-graph-1.18bfe0dd44a9805e namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: 9a8c835f-6bf9-48ff-9c99-5e783ebe1ed3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: da6fad0a-4757-42ab-9f73-252676507953 kind: Event lastTimestamp: "2026-07-07T02:28:40Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-07T02:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:40Z" name: isvc-sklearn-graph-1.18bfe0f5029d5ac4 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 12871c97-31ca-4dea-a4c1-3f12ad1bc662 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:26:58.772996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dd5a8410ea namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 1ef4ab53-2d24-42d6-a830-9461e7f96da5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dd634b1591 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 17b92391-4e04-4fd1-b323-c54d4f185d3a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:26:59Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dda02a49ba namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: bd3bd7fe-7b1b-436a-9f79-245e091585f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:f221c928a0bb1ba5e4c45ae19b5fe18d4abac3f5b57f42edecf256c4242d4d40" already present on machine metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:59Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dda2781d97 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: af90fb9a-f549-4405-8db0-c82ccb7707c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T02:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:00Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dda7b1977b namespace: kserve-ci-e2e-test resourceVersion: "14587" uid: 71443908-e930-4cd2-b4a3-f88fbc375eaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T02:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:00Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dda85d87eb namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: ecdbe5f3-7e68-4215-bda7-bb49073979b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1702" metadata: creationTimestamp: "2026-07-07T02:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:04Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0dec56077a3 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: d5271dbf-56e7-4418-80c2-3899d2eeb7cc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:16Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1702" in 11.763s (11.763s including waiting). Image size: 1555967903 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:16Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e1828687d8 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 65ab06af-f2df-4ff7-9105-061f3f1aad57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:16Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e1872353fa namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 3bf10e0f-4665-4b26-93ea-3f0a9b61b94e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:16Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e187dddef5 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 67c1ef99-94e0-4ba2-8be1-cb427f41978d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:16Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T02:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:16Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e188379d4a namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: 4ce56926-6ddc-4b6e-946f-826c6d99da93 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.251s (3.251s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:19Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e24a0bc9c5 namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: 973ba6c0-ef50-49e9-9006-246d37e6570f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:20Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e24ea90e2e namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: ef2f65f8-d8be-4ee9-b030-bb0ca38a7f48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:27:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:20Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e24f4c711e namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: 23a149d9-0d3d-47f3-8ae7-8c801c031fa8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:29:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:21Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0e286c5b725 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: ab1fa12f-2e1a-475c-ab1c-aabc0500111b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0fde61fcff0 namespace: kserve-ci-e2e-test resourceVersion: "16113" uid: 44f3c2ba-7999-41d4-bc81-a601c71bfabc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0fde6206873 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 9f0a8406-4a42-47a6-ae75-090c3b15ee50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 2635b90c-7986-46a8-a45a-1a209eb475fb kind: Event lastTimestamp: "2026-07-07T02:29:21Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:21Z" name: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn.18bfe0feb34cbf14 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 7774d40c-0fdd-4def-a6b1-75aaaa59d432 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-ccccddb65 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: c3eafe42-c98d-4a4c-a436-2e342222479f kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-ccccddb65-rwvgn' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-2-predictor-ccccddb65.18bfe0dd5a1708e5 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: c5fa4b3e-5f74-4b3a-aec5-39326cb7c0dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: b61a0f42-c191-40ff-b543-528a32536ab1 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-ccccddb65 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: isvc-sklearn-graph-2-predictor.18bfe0dd5773e481 namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 5894b45a-ee94-47ce-8382-6277b2792600 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 9117b3c0-37d9-45dd-961f-1c2cb05b5c2e kind: Event lastTimestamp: "2026-07-07T02:27:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:43Z" name: isvc-sklearn-graph-2-predictor.18bfe0e0d5d2af35 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 07d633f5-b4c3-4fab-ab75-699a073e0844 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 9117b3c0-37d9-45dd-961f-1c2cb05b5c2e kind: Event lastTimestamp: "2026-07-07T02:27:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:43Z" name: isvc-sklearn-graph-2-predictor.18bfe0e0d63be9f6 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: 66ad96c8-40e9-4fdc-a2fd-7d1026ddff97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 9117b3c0-37d9-45dd-961f-1c2cb05b5c2e kind: Event lastTimestamp: "2026-07-07T02:28:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:58Z" name: isvc-sklearn-graph-2-predictor.18bfe0eb52e432f3 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 5a452ebf-54de-4d81-a8bf-37749f1bfdd7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 9117b3c0-37d9-45dd-961f-1c2cb05b5c2e kind: Event lastTimestamp: "2026-07-07T02:28:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:58Z" name: isvc-sklearn-graph-2-predictor.18bfe0eb53c2d435 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 05a65ef9-3f7c-42e1-a717-c500dc8a1814 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 28a66bcf-f6a5-44e2-908d-bac1cbceed6a kind: Event lastTimestamp: "2026-07-07T02:28:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:41Z" name: isvc-sklearn-graph-2.18bfe0dd577697d5 namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 81af2260-1571-4f3b-9c9c-cab0060e97dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 28a66bcf-f6a5-44e2-908d-bac1cbceed6a kind: Event lastTimestamp: "2026-07-07T02:28:41Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-07T02:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:41Z" name: isvc-sklearn-graph-2.18bfe0f53e674ef2 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 8d278045-4fef-452d-b5aa-20fac7431cf9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:26:58.676457Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-vbstl to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:26:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0dd54c30dee namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 8120f8dd-92e1-46cb-818d-09b6bd9df156 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:26:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0dd6e834919 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 34ae1f41-d4c4-43d8-8074-d897ac00e672 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:f221c928a0bb1ba5e4c45ae19b5fe18d4abac3f5b57f42edecf256c4242d4d40" already present on machine metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0dd704fe46a namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: ede9fdd7-787a-49de-a7b9-d2d2dc100b0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0dd732a2f20 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 9fad9610-b686-43d6-be69-72581178c161 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:26:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T02:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0dd7395f0c7 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 534c5d7a-43b6-465d-9e2f-80b7d21a807c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:04Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-07T02:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0dec5236ab5 namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: b956cb93-4bbb-4036-877d-2e9cee6adec0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:37Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 32.215s (32.215s including waiting). Image size: 1305293051 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e6455845e4 namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: d1964417-ef47-497f-af32-5f0d3d29fa02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e64bf64901 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: c249cb52-fe4f-4bf4-9ab6-3f2a69508602 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e64d3b31bb namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: a527fda2-f136-4f14-ae16-180c6c5092ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e64d522191 namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: b3e3c4db-6e7e-4563-ba0f-d069faefb542 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e653c0c525 namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 2676223f-87fa-4851-b8ff-13fc70dda6c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:27:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e6558a5f8d namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: b8206425-5aab-4ad2-be54-ba938feee826 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T02:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:28:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0e681d42164 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: ea43594e-edda-4a23-976d-0af6891b322c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0fde28fb331 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 0b14cab5-9e4f-4c1b-839a-11fa4205e11a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0fde2904399 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 3555431b-1fdf-4530-bd7c-80a1451f120c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-vbstl namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 636ca02f-d3be-454e-81ab-557b10492a29 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-vbstl.18bfe0fe061b8751 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 3972fb4b-7359-4cc2-95ef-bf52452a9fcc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 279c1bbe-6999-4899-b119-48447dca6163 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-vbstl' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: isvc-xgboost-graph-predictor-555fc69d56.18bfe0dd545527b8 namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: ae602092-31d8-4ce4-b64f-d82a0a093f1c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 7e7f4c9c-27c6-40a1-a81e-4b7e291d23d5 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: isvc-xgboost-graph-predictor.18bfe0dd50c7ac94 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 3ddc48f6-ace0-4841-8848-737f8b4d3940 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: dde28c81-c938-4590-ac98-43d85bbf3049 kind: Event lastTimestamp: "2026-07-07T02:28:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:13Z" name: isvc-xgboost-graph-predictor.18bfe0e0cfa81047 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: e4649374-7724-4167-87f1-112cfd03efc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: dde28c81-c938-4590-ac98-43d85bbf3049 kind: Event lastTimestamp: "2026-07-07T02:28:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:13Z" name: isvc-xgboost-graph-predictor.18bfe0e0d03d2f5d namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 09ec5f69-b071-472f-ad97-ab01c93619b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: dde28c81-c938-4590-ac98-43d85bbf3049 kind: Event lastTimestamp: "2026-07-07T02:29:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:13Z" name: isvc-xgboost-graph-predictor.18bfe0eb4d2ce8ba namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 754a22d0-4c72-4d6b-b3cf-ac596948414b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: dde28c81-c938-4590-ac98-43d85bbf3049 kind: Event lastTimestamp: "2026-07-07T02:29:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:13Z" name: isvc-xgboost-graph-predictor.18bfe0eb4e3db6ea namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 635bc83c-dcae-42fa-a671-c553e12571f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 2c8e6647-5880-4297-b9a2-a37d8b99e8d2 kind: Event lastTimestamp: "2026-07-07T02:28:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:44Z" name: isvc-xgboost-graph.18bfe0dd50cfcb08 namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 36070502-be82-4a5b-b1e9-c24372f8fccc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 2c8e6647-5880-4297-b9a2-a37d8b99e8d2 kind: Event lastTimestamp: "2026-07-07T02:28:44Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-07T02:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:44Z" name: isvc-xgboost-graph.18bfe0f5e9189f9d namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: db243e19-e4d6-48e2-8596-ecbf002389f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:29:08.287127Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7c56457f57-rzgnn to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fb8228e954 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 3e88e24b-5a81-479a-a212-230c09a3fc4b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fb8abd19bc namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 2b03adc5-3793-4679-a241-b0a66acd36ad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:09Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:29:09Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fbbfdea1cb namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: e3b2d1c8-294c-4753-a48f-b05957a19f25 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:09Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fbc2426698 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 19e2e409-7efc-42ed-a671-b1011ad80fff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:09Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-07T02:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:09Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fbc50bd23d namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 7a7897fe-ee30-4489-a4db-92b8ffaeaaf3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:09Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-07T02:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:09Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fbc58081a2 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 73accc59-3524-4f47-8792-c20ddf6c9f2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fddc1c4ff6 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 07329f1d-b93b-4ba7-ad69-d826914bf351 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7c56457f57-rzgnn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: df9ad0dd-a68c-4b6d-b353-f7d86c6c5439 kind: Event lastTimestamp: "2026-07-07T02:29:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:46Z" name: model-chainer-7c56457f57-rzgnn.18bfe0fe77e8e771 namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 8c2dbb69-df46-460c-b378-98f37900c336 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7c56457f57 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: 74d85543-5ccb-4d19-8694-5ba5762d2da1 kind: Event lastTimestamp: "2026-07-07T02:29:08Z" message: 'Created pod: model-chainer-7c56457f57-rzgnn' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer-7c56457f57.18bfe0fb81c5e944 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 3e87ef30-e9fa-4f2f-82bd-c23019972a0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: c143aa9a-7a45-46fb-89aa-620bd76d8cbd kind: Event lastTimestamp: "2026-07-07T02:29:08Z" message: Scaled up replica set model-chainer-7c56457f57 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer.18bfe0fb80a29f75 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 6b2e9de6-d5a3-44da-8e10-b6e7d51b4b22 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: ca7d3a13-8d48-41ed-8479-707c4102dbfa kind: Event lastTimestamp: "2026-07-07T02:29:08Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer.18bfe0fb863edc3a namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: ac40985b-6410-491d-8919-335a54fe2419 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: ca7d3a13-8d48-41ed-8479-707c4102dbfa kind: Event lastTimestamp: "2026-07-07T02:29:08Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer.18bfe0fb8953aed0 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: eacfd547-dc7b-4a08-8195-4dda304d1761 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: ca7d3a13-8d48-41ed-8479-707c4102dbfa kind: Event lastTimestamp: "2026-07-07T02:29:08Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:08Z" name: model-chainer.18bfe0fb89541366 namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 5407cfc8-c424-40aa-a5c5-5774b19b026c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:39:33.582280Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-524cd-55f58ffb86-hvhqx to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:39:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:39:33Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18d18a7af02 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: ea9e5b20-6d74-49f0-9896-7e7a8c62156f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:33Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:39:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-524cd-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:34Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18d1f7aa44f namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: c29dc67c-d0a7-4a85-9b5f-40e9f74e69f0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:39:35Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:39:35Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18d8b4f0276 namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: a6e2e64d-4b16-45bb-8efe-ff0126f774e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-524cd} kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:39:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:35Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18d8c8af011 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 2ec677a2-74f6-46ca-bd56-93adb8782533 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-524cd} kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:39:35Z" message: 'Created container: sequence-graph-524cd' metadata: creationTimestamp: "2026-07-07T02:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:35Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18d8f4ea36a namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 012003ac-94e2-477d-9cc2-a09242c58193 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-524cd} kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:39:35Z" message: Started container sequence-graph-524cd metadata: creationTimestamp: "2026-07-07T02:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:35Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18d8fbce619 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 9b280e43-7cc3-40c8-8493-a9785ac16b54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-524cd} kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Stopping container sequence-graph-524cd metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:43Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe18f71fb7b48 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 0c8ffde3-7c27-4694-8414-c965f09362af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-524cd} kind: Pod name: sequence-graph-524cd-55f58ffb86-hvhqx namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: fe76f6bf-5c18-4c9d-8142-47c5318c59d3 kind: Event lastTimestamp: "2026-07-07T02:40:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:12Z" name: sequence-graph-524cd-55f58ffb86-hvhqx.18bfe19050ba9556 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: b76928ed-3fba-4a5c-86fd-f8c1192bf2e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-524cd-55f58ffb86 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 7c62990d-6b00-4091-b862-b024472b6d58 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: 'Created pod: sequence-graph-524cd-55f58ffb86-hvhqx' metadata: creationTimestamp: "2026-07-07T02:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:33Z" name: sequence-graph-524cd-55f58ffb86.18bfe18d18368f0a namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 4de3556a-a4d3-4d5a-98b6-c81cec95acff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-524cd namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: e860d5ef-644f-44b0-96fb-53ce85f383c2 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: Scaled up replica set sequence-graph-524cd-55f58ffb86 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:33Z" name: sequence-graph-524cd.18bfe18d177de4e8 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 2eb27314-b44e-4dc4-bf00-345b2c337edb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-524cd namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: eb3bcbe8-09fb-43c7-ac16-68894e76d2d0 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: InferenceGraph [sequence-graph-524cd] is Ready metadata: creationTimestamp: "2026-07-07T02:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:33Z" name: sequence-graph-524cd.18bfe18d1cb73519 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 153187f7-00e0-4cbc-b6bb-e32e05dd0924 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:40:43.927027Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7c238-5c449cbc68-zsbrd to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:40:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:40:43Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe19d79874fb9 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 6148316f-5f59-4baf-a646-222439ca2019 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:44Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:40:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-7c238-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:44Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe19d80a21a8c namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: c9f7828b-afc7-4ed7-ac6a-15ef7daf675d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:44Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:40:44Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:40:44Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe19db7164737 namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: ffdcba7d-55fe-4c99-990d-f8b4f71f66c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c238} kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:40:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:44Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe19db891713d namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: 7e56fce3-f57b-40a2-9faf-47ee54dde41e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c238} kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:40:45Z" message: 'Created container: sequence-graph-7c238' metadata: creationTimestamp: "2026-07-07T02:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:45Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe19dbc7467aa namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: 4aeec319-6e28-4725-a3dc-a3019be5a3a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c238} kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:40:45Z" message: Started container sequence-graph-7c238 metadata: creationTimestamp: "2026-07-07T02:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:40:45Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe19dbceea5e1 namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: c484e4fb-970d-4b75-88f0-8f3e0dd9d596 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c238} kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Stopping container sequence-graph-7c238 metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:58Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe210a6cf9107 namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 6586c4b2-2aea-4792-9354-d076ec9957fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c238} kind: Pod name: sequence-graph-7c238-5c449cbc68-zsbrd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 9605e293-a6b5-44a0-880b-04bd9050d607 kind: Event lastTimestamp: "2026-07-07T02:49:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:26Z" name: sequence-graph-7c238-5c449cbc68-zsbrd.18bfe2115d97cf36 namespace: kserve-ci-e2e-test resourceVersion: "26254" uid: 807c835b-5c1f-430f-aa00-324964a9f079 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7c238-5c449cbc68 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: 327f9d3e-2384-4442-b3e4-e39072d42b3c kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: 'Created pod: sequence-graph-7c238-5c449cbc68-zsbrd' metadata: creationTimestamp: "2026-07-07T02:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:43Z" name: sequence-graph-7c238-5c449cbc68.18bfe19d78feb4a2 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: 716017b2-75d9-4e1e-8e4d-6c89cd49bf60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7c238 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 111c65fa-9368-4fb4-93fb-b712f0108898 kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: Scaled up replica set sequence-graph-7c238-5c449cbc68 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:43Z" name: sequence-graph-7c238.18bfe19d7848fe3a namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: f696b9b6-a266-46aa-8143-75e70eaaabcc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7c238 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: 3b0ff52d-16e5-490f-8a82-c8fffc9a4d28 kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: InferenceGraph [sequence-graph-7c238] is Ready metadata: creationTimestamp: "2026-07-07T02:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:43Z" name: sequence-graph-7c238.18bfe19d7d808f9c namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 3ea35cb2-4e19-45a7-a128-79db870dd49f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7c238 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: c74be928-775d-47fd-8b0e-a823e4bac61f kind: Event lastTimestamp: "2026-07-07T02:41:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:41:13Z" name: sequence-graph-7c238.18bfe1a0f6ba9411 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 5aeade25-c8f4-491d-8d68-e3f14f2ecad9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7c238 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: c74be928-775d-47fd-8b0e-a823e4bac61f kind: Event lastTimestamp: "2026-07-07T02:41:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:41:13Z" name: sequence-graph-7c238.18bfe1a0f7185489 namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 3a9deca4-5785-4c9f-9ec0-ba2a59f0dd78 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:30:18.607764Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-821ea-f6c64f499-77srd to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:30:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:30:18Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe10be19895ac namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 35139b83-3459-42b9-aa49-06c6fcff7632 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:18Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:30:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-821ea-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:30:18Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe10becffe3fc namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 29bc82dc-62be-4c9e-a87b-a14e6c8fe382 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:19Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:30:19Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:30:19Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe10c1e9e7715 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 91154aef-36e9-43e4-b707-f7253c516b76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-821ea} kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:30:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:30:19Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe10c1fea2b36 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 2cfac095-db43-428a-a5f6-42646a0da5da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-821ea} kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:30:19Z" message: 'Created container: sequence-graph-821ea' metadata: creationTimestamp: "2026-07-07T02:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:30:19Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe10c22bfad15 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 61bc187a-43aa-4b15-ac5c-cea369a7bb64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-821ea} kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:30:19Z" message: Started container sequence-graph-821ea metadata: creationTimestamp: "2026-07-07T02:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:30:19Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe10c232ebe31 namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: ebecde75-1aec-486f-907b-bf6abc4b5c56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-821ea} kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Stopping container sequence-graph-821ea metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe17f15e1be72 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 37b2e09b-2604-4b83-b762-78361019931c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-821ea} kind: Pod name: sequence-graph-821ea-f6c64f499-77srd namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 560669b5-f71a-48ed-bbf4-d1d6036b3341 kind: Event lastTimestamp: "2026-07-07T02:39:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:01Z" name: sequence-graph-821ea-f6c64f499-77srd.18bfe17fc0bab7d0 namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 94453a6a-3d0e-45c1-905f-6e9f8ad6c556 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-821ea-f6c64f499 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 4738f515-4580-434d-ba08-1d04ccab0b46 kind: Event lastTimestamp: "2026-07-07T02:30:18Z" message: 'Created pod: sequence-graph-821ea-f6c64f499-77srd' metadata: creationTimestamp: "2026-07-07T02:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:18Z" name: sequence-graph-821ea-f6c64f499.18bfe10be0f24746 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 17a3dd5f-ef67-4679-aba2-097a8ddcc9c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-821ea namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: e2491ebb-ef9c-47ed-a2fa-9fdd0463974d kind: Event lastTimestamp: "2026-07-07T02:30:18Z" message: Scaled up replica set sequence-graph-821ea-f6c64f499 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:18Z" name: sequence-graph-821ea.18bfe10be0480fea namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: b911e17c-5113-41a9-8ba1-a1e2e73242aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-821ea namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 12d21acf-7fc4-4ddf-9e44-023c6fb8b900 kind: Event lastTimestamp: "2026-07-07T02:30:18Z" message: InferenceGraph [sequence-graph-821ea] is Ready metadata: creationTimestamp: "2026-07-07T02:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:30:18Z" name: sequence-graph-821ea.18bfe10be62730d8 namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: d3f99c79-adc9-437c-a31c-3bfb9de97705 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:30:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-821ea namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: c6bc9d37-6cb6-44fe-92ff-c14adec986a6 kind: Event lastTimestamp: "2026-07-07T02:30:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:48Z" name: sequence-graph-821ea.18bfe10f6021ae07 namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: b76ebd83-2c0b-43a6-a17d-69b8f89bcc5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:30:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-821ea namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: c6bc9d37-6cb6-44fe-92ff-c14adec986a6 kind: Event lastTimestamp: "2026-07-07T02:30:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:30:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:48Z" name: sequence-graph-821ea.18bfe10f61330772 namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 362bd90e-ed4b-4a2e-924f-6a9c0193e87a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:49:39.013973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4831c-7579877bcd-79mz6 to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:49:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:49:39Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21a0f32284e namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: c283c173-3a49-4a9c-bfac-ab59af4ddc7b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:49:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-4831c-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:39Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21a2008d0c4 namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: a5982937-35af-4b26-aa05-f5aed6a7c038 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:40Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:49:40Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:49:40Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21a4c7b9d2d namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: 8c1b9cfc-ec63-458d-aba8-686781a1140e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4831c} kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:49:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:40Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21a4dc96cfa namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 20020b06-3474-4cb1-858d-c69ba115ce0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4831c} kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:49:40Z" message: 'Created container: splitter-graph-4831c' metadata: creationTimestamp: "2026-07-07T02:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:40Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21a50d31c86 namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: f13079b1-049e-46f0-8ebb-dd837ad9439d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4831c} kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:49:40Z" message: Started container splitter-graph-4831c metadata: creationTimestamp: "2026-07-07T02:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:40Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21a5146d69d namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 9fdc06eb-269e-4627-80b4-b38c83b3a389 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4831c} kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Stopping container splitter-graph-4831c metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21c68d1969f namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: b31cd908-a30e-4a80-ad26-11eba1c3d088 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4831c} kind: Pod name: splitter-graph-4831c-7579877bcd-79mz6 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 931c7f1c-30fc-45e0-94ae-07ee78cbb4c7 kind: Event lastTimestamp: "2026-07-07T02:50:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:50:16Z" name: splitter-graph-4831c-7579877bcd-79mz6.18bfe21d03fac0f8 namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: 1dcfbaba-49b4-49be-b6cd-b4b2a0465fdf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4831c-7579877bcd namespace: kserve-ci-e2e-test resourceVersion: "26360" uid: f994ca0c-47d6-4892-a745-d7e8acaaef4a kind: Event lastTimestamp: "2026-07-07T02:49:39Z" message: 'Created pod: splitter-graph-4831c-7579877bcd-79mz6' metadata: creationTimestamp: "2026-07-07T02:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:39Z" name: splitter-graph-4831c-7579877bcd.18bfe21a0eb684c7 namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 1c621e27-85af-4702-a034-df2fe96ad655 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4831c namespace: kserve-ci-e2e-test resourceVersion: "26358" uid: 98ab118a-54b3-4e96-b4c2-fc01f2df7dcc kind: Event lastTimestamp: "2026-07-07T02:49:38Z" message: Scaled up replica set splitter-graph-4831c-7579877bcd from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:38Z" name: splitter-graph-4831c.18bfe21a0dea0323 namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 55504a8b-f02a-4010-ac94-8bb30c89f6e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4831c namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 31003aed-fa6a-4e51-a289-057c02cc94b1 kind: Event lastTimestamp: "2026-07-07T02:49:39Z" message: InferenceGraph [splitter-graph-4831c] is Ready metadata: creationTimestamp: "2026-07-07T02:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:39Z" name: splitter-graph-4831c.18bfe21a131a43ea namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 49d49f8e-338a-4be0-b700-0e6789d28777 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:50:49.358539Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-8f974-869db4d598-v5srq to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe22a700f0abb namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: 8bed2679-1870-4e11-84e8-d13cc4bc5670 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27178" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe22a89595396 namespace: kserve-ci-e2e-test resourceVersion: "27189" uid: 642e5bef-411f-44b0-abcb-fc15686c284d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8f974} kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe22a8a923019 namespace: kserve-ci-e2e-test resourceVersion: "27191" uid: df890ec9-31be-4de2-96b6-72f21214beb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8f974} kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: 'Created container: splitter-graph-8f974' metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe22a8d9b757c namespace: kserve-ci-e2e-test resourceVersion: "27192" uid: f85e5250-ad70-4ee8-8001-beccb0037e4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8f974} kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: Started container splitter-graph-8f974 metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe22a8e0be8dc namespace: kserve-ci-e2e-test resourceVersion: "27193" uid: e49e2014-d563-474a-8fb7-a116502f511b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8f974} kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: "2026-07-07T02:59:03Z" message: Stopping container splitter-graph-8f974 metadata: creationTimestamp: "2026-07-07T02:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:03Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe29d98db256d namespace: kserve-ci-e2e-test resourceVersion: "30636" uid: c710ab6b-2fce-441e-8880-cca0754bd2f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:59:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8f974} kind: Pod name: splitter-graph-8f974-869db4d598-v5srq namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: ef650321-7451-4b33-83bb-3914db209272 kind: Event lastTimestamp: "2026-07-07T02:59:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:30Z" name: splitter-graph-8f974-869db4d598-v5srq.18bfe29e111f4bb6 namespace: kserve-ci-e2e-test resourceVersion: "30867" uid: b460c7df-ed31-4be3-9acc-db3e299ad9ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-8f974-869db4d598 namespace: kserve-ci-e2e-test resourceVersion: "27164" uid: 8b114573-759e-40ba-bd5c-a6f3657a2a9d kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: 'Created pod: splitter-graph-8f974-869db4d598-v5srq' metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974-869db4d598.18bfe22a6f9b460f namespace: kserve-ci-e2e-test resourceVersion: "27172" uid: 961d86c6-3115-4936-afc1-a96cb4e232a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27162" uid: 3f052b57-e5d2-4cd0-93f0-df326e55567c kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: Scaled up replica set splitter-graph-8f974-869db4d598 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974.18bfe22a6e92d7a5 namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: a857ad17-76b4-41f9-a792-06162bf2e478 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: b9ead6e2-0948-42ff-a7aa-3c9169ffd27a kind: Event lastTimestamp: "2026-07-07T02:50:49Z" message: InferenceGraph [splitter-graph-8f974] is Ready metadata: creationTimestamp: "2026-07-07T02:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:50:49Z" name: splitter-graph-8f974.18bfe22a726b7e00 namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: 3f22e708-536b-4439-9bc4-2388419a1ced reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:51:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27163" uid: 73481bce-f5a2-4077-a855-c61bcb73475d kind: Event lastTimestamp: "2026-07-07T02:51:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:51:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:04Z" name: splitter-graph-8f974.18bfe22ded13a7d3 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: ae87967b-0a61-4856-8b85-d99c7f71523c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:51:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27163" uid: 73481bce-f5a2-4077-a855-c61bcb73475d kind: Event lastTimestamp: "2026-07-07T02:51:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:51:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:04Z" name: splitter-graph-8f974.18bfe22ded7c337f namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: 1368eab3-c11d-482f-994d-c47d750cb9a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:51:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 73481bce-f5a2-4077-a855-c61bcb73475d kind: Event lastTimestamp: "2026-07-07T02:51:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:19Z" name: splitter-graph-8f974.18bfe2316c4b53c4 namespace: kserve-ci-e2e-test resourceVersion: "27413" uid: f371637a-22bd-4bd0-9413-f09001f1b265 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:51:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 73481bce-f5a2-4077-a855-c61bcb73475d kind: Event lastTimestamp: "2026-07-07T02:51:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:19Z" name: splitter-graph-8f974.18bfe2316c9e4efd namespace: kserve-ci-e2e-test resourceVersion: "27414" uid: 11435167-2d75-40a9-b57b-591256da1e25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:26:58.174851Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2346e-predictor-f9b49f577-z2m5r to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:26:58Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0dd36dd0b99 namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: 2b478a1f-c83a-49bd-9ff4-8a64dc48ea9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:26:58Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0dd51f07127 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: efd457fd-a9de-4b5e-8d1a-7d0716c16a33 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:26:58Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0dd53b51f8e namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: 00617471-d9e0-4473-9252-e9ecb75f68eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" in 17.013s (17.013s including waiting). Image size: 1335616093 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e149ce4115 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 80bd8c3c-799c-4954-a631-740cb6f2cc1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e14ef935ef namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 7162ff9c-0a13-4cbe-a15d-17c9e7bb6c27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e14f8e8ff7 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 581b5c8c-b084-473e-a24d-f870355a71aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T02:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:15Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e14fa54886 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 018d7d8b-1aee-43b2-8a89-4e2c657607b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.19s (4.19s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T02:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:19Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e2496e0e3c namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: c0dc1938-b61f-496e-96d8-8be6c8d21bc2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:20Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e24e83712a namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 73ca6781-3d98-4c01-843b-4619bd2e67f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:27:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:27:20Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e24f1e19ac namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: 42b1e6cc-743b-4b68-a020-8c20ee29bbde reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:28:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:07Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0e28678df8c namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 8c2c9663-8d85-4e7f-a36f-70db4ef9325a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0f49dffe9b0 namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: cab3cb41-e0f0-4147-af08-84f322809250 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0f49e007430 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: c1c013fc-71f6-47a4-9772-308107c1481d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 5392cc5b-e830-40d9-b4d1-e673cd917124 kind: Event lastTimestamp: "2026-07-07T02:28:41Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:41Z" name: success-200-isvc-2346e-predictor-f9b49f577-z2m5r.18bfe0f5632451c1 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 5bb6e9e7-f718-4923-8a54-638a354ba9b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2346e-predictor-f9b49f577 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 6b30a8a0-e5c9-4efd-95a6-ea1ccd8fe193 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: 'Created pod: success-200-isvc-2346e-predictor-f9b49f577-z2m5r' metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: success-200-isvc-2346e-predictor-f9b49f577.18bfe0dd366eb1f4 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: af5d65fd-5e13-4978-8d39-541bc2a31edd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: c570ed3f-882d-4e58-8585-8a43be31ae23 kind: Event lastTimestamp: "2026-07-07T02:26:58Z" message: Scaled up replica set success-200-isvc-2346e-predictor-f9b49f577 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:26:58Z" name: success-200-isvc-2346e-predictor.18bfe0dd33252c8d namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 6f938aa9-4a27-4548-883f-3d1fe5cd1371 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 57dd56db-ad71-4bea-a22f-9d1e5b55da62 kind: Event lastTimestamp: "2026-07-07T02:27:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:28Z" name: success-200-isvc-2346e-predictor.18bfe0e0b8e0022d namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 10b0f529-1e57-49ea-b250-372f75ee1953 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:27:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 57dd56db-ad71-4bea-a22f-9d1e5b55da62 kind: Event lastTimestamp: "2026-07-07T02:27:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:27:28Z" name: success-200-isvc-2346e-predictor.18bfe0e0b9710566 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: c7b62b1d-69ff-4e6a-9020-e26a673673ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:27:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 57dd56db-ad71-4bea-a22f-9d1e5b55da62 kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:28Z" name: success-200-isvc-2346e-predictor.18bfe0e7b8a2a4ec namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: b9dce1a0-09a8-4340-a03e-9d2e1d105197 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:27:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2346e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 57dd56db-ad71-4bea-a22f-9d1e5b55da62 kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:28Z" name: success-200-isvc-2346e-predictor.18bfe0e7b9b4c344 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 95c8dc54-c794-4f04-868d-2c56e42030ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:26:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2346e namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: bd32c272-420e-4753-882b-369aeda90a83 kind: Event lastTimestamp: "2026-07-07T02:28:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:17Z" name: success-200-isvc-2346e.18bfe0dd395588b1 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 25e3534e-5a19-48d4-bb6e-f117364bd432 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2346e namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: bd32c272-420e-4753-882b-369aeda90a83 kind: Event lastTimestamp: "2026-07-07T02:28:17Z" message: InferenceService [success-200-isvc-2346e] is Ready metadata: creationTimestamp: "2026-07-07T02:28:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:17Z" name: success-200-isvc-2346e.18bfe0ef9e571bf9 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 5ed11fdf-2cc0-4128-887b-ad0c8e54fa84 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:28:38.757373Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4a20c8c20 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: d582b4a0-df33-4856-8ef4-8612622e90b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4bc38f632 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 3f1b13e6-afb2-4120-af52-42dac7502f5f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4beaa8b5f namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 78efaec9-b8aa-476e-8b7a-f85eb1f19b08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4c25cc39f namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 923cc4df-1c57-46f4-bb12-de7bbb0ffc7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4c2c99937 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 665bbd2b-1fb3-439a-ac2a-a0e02cada9c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4c2d462e3 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 749cee18-a7de-40c1-b9cb-7bdd86018cc1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4c69e4967 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 63cecdb6-7756-4828-9951-dc552ac2ad03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:28:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:39Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4c70ddaf3 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 6cbc934f-968f-4f9d-ab6c-f3d5eb048ee5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T02:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:38:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:05Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe0f4c85d99b8 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: d2200a6f-4b8b-487a-b391-f4a343be82c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe17820745030 namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 7b1c0fb0-b70c-40f4-bd63-e03c37f99cdd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe17820750050 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: afd873b6-f6e1-4be0-a40f-993645d4c05b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: bd23a8eb-edd5-4d82-9523-36e7aff2e825 kind: Event lastTimestamp: "2026-07-07T02:38:05Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:05Z" name: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7.18bfe17891022c67 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: c42fe1ee-7e13-4788-8d86-85a13b544902 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3ea7f-predictor-7f4997b577 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: da1ac4d9-d96d-4085-8f09-e5575bd69386 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: 'Created pod: success-200-isvc-3ea7f-predictor-7f4997b577-vs7b7' metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-3ea7f-predictor-7f4997b577.18bfe0f4a0edfa26 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 379aadcb-3290-415b-ba8c-b3ea9995f0df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 885e75d6-1e07-46b4-97ef-ddd0539b5429 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Scaled up replica set success-200-isvc-3ea7f-predictor-7f4997b577 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-3ea7f-predictor.18bfe0f49acb08b3 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 1b077d68-1100-4a7b-91cf-ea2ed93a5849 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 03105a9a-2a6c-4074-9ecd-94347ff43789 kind: Event lastTimestamp: "2026-07-07T02:28:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:53Z" name: success-200-isvc-3ea7f-predictor.18bfe0f81944bef3 namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: d7e287d0-3c8c-47af-97f4-a5df5449edaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 03105a9a-2a6c-4074-9ecd-94347ff43789 kind: Event lastTimestamp: "2026-07-07T02:28:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:53Z" name: success-200-isvc-3ea7f-predictor.18bfe0f819b2fa5c namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: c508ce08-512b-4d04-8e7a-b6bf900e5ac7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 03105a9a-2a6c-4074-9ecd-94347ff43789 kind: Event lastTimestamp: "2026-07-07T02:30:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:08Z" name: success-200-isvc-3ea7f-predictor.18bfe0fb99036f47 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 93c0867f-660d-4e00-b10c-7830b1037358 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:29:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3ea7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 03105a9a-2a6c-4074-9ecd-94347ff43789 kind: Event lastTimestamp: "2026-07-07T02:30:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:08Z" name: success-200-isvc-3ea7f-predictor.18bfe0fb998eac10 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 080468dd-4b56-4879-97ac-ab60c924b6dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: ae4f3544-5f6c-4d86-a213-2a69d0fe0949 kind: Event lastTimestamp: "2026-07-07T02:29:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:35Z" name: success-200-isvc-3ea7f.18bfe0f49b8b2d13 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: 7416c0ae-d887-4087-9cd4-7b147c09a8fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: ae4f3544-5f6c-4d86-a213-2a69d0fe0949 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-3ea7f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3ea7f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-3ea7f.18bfe0f49c470a99 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: 0123961b-4197-4107-b737-f11b9685b81d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: ae4f3544-5f6c-4d86-a213-2a69d0fe0949 kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3ea7f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:38Z" name: success-200-isvc-3ea7f.18bfe0f49c476f36 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 2d043838-c877-4de1-b17b-ec7beb1e8531 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: ae4f3544-5f6c-4d86-a213-2a69d0fe0949 kind: Event lastTimestamp: "2026-07-07T02:29:35Z" message: InferenceService [success-200-isvc-3ea7f] is Ready metadata: creationTimestamp: "2026-07-07T02:29:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:35Z" name: success-200-isvc-3ea7f.18bfe101d7b38ad9 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 7c9aa324-0b46-4e87-b4a9-53c45dcd7e98 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:48:28.951794Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4831c-predictor-854cc4c887-64bp4 to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209bf2a38b0 namespace: kserve-ci-e2e-test resourceVersion: "25558" uid: e5545c36-e556-418e-999e-536768a7f1b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25565" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209d9ab13d2 namespace: kserve-ci-e2e-test resourceVersion: "25602" uid: 8580eb46-4509-4450-a8f0-95aef8a5e98d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209db3c27e3 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: e3e31701-6653-41df-a102-ee214584bd00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209de663aaf namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 1c0d9df2-dfaa-4c77-8c0c-494ea77a6bd0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209dee3dfcd namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 352ddb2d-37c8-4cb4-b2f8-a229f09fbbb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209deef89a4 namespace: kserve-ci-e2e-test resourceVersion: "25608" uid: 83584116-2f17-4ac7-ad6d-cceefd359a46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209e26f772f namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: 5ee00804-81f3-457b-bc40-613415124870 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:48:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:29Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209e2eef3eb namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: 1f5dea9a-ecbc-4b11-b1c1-2ea0d864f20a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:49:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:15Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe209ec93ee6b namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: e7d67c5a-e7ef-441b-b0bd-24ece34721cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe21c6fa79c34 namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 8a0515ea-d409-4772-9bef-dd330d9b9882 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe21c6fa82681 namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 33004d70-088d-4ca5-93ae-80b3ef831f93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4831c-predictor-854cc4c887-64bp4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: a75f06b3-7b59-4a23-9b2e-e4c08af9b39a kind: Event lastTimestamp: "2026-07-07T02:49:50Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:50Z" name: success-200-isvc-4831c-predictor-854cc4c887-64bp4.18bfe21cc8f00b43 namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 6ee0067d-7347-4cee-bec5-ecc0cb5ce192 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4831c-predictor-854cc4c887 namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: 151a7bb8-4508-4780-806a-3422851a3277 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: 'Created pod: success-200-isvc-4831c-predictor-854cc4c887-64bp4' metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-4831c-predictor-854cc4c887.18bfe209bed3f49f namespace: kserve-ci-e2e-test resourceVersion: "25548" uid: ea1f7df4-905f-4f2c-987c-b56cf18d9c11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: 2c75d2b9-f4e2-4882-9101-cb37f19dd5f2 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Scaled up replica set success-200-isvc-4831c-predictor-854cc4c887 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-4831c-predictor.18bfe209bb306226 namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: b9e48430-d146-45bb-93ce-e10779453e8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25527" uid: 2808662b-1766-443b-9a6e-aa88fe59d896 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-4831c-predictor.18bfe20d39b441fe namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 24c42316-40f8-4cbd-963c-8029b68df056 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25527" uid: 2808662b-1766-443b-9a6e-aa88fe59d896 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-4831c-predictor.18bfe20d3a1e34a7 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 54ea6f92-b918-455b-a7a2-7ba6213fbd56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: 2808662b-1766-443b-9a6e-aa88fe59d896 kind: Event lastTimestamp: "2026-07-07T02:49:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:43Z" name: success-200-isvc-4831c-predictor.18bfe214378d556b namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: ef25949b-57b5-4b54-a407-8d5fc5056443 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4831c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: 2808662b-1766-443b-9a6e-aa88fe59d896 kind: Event lastTimestamp: "2026-07-07T02:49:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:43Z" name: success-200-isvc-4831c-predictor.18bfe21437ee5f84 namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: ed1a96d1-0fc8-4cd6-94cd-58da24bfb48f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "25503" uid: 04f29d90-a3c4-45e8-9c6a-5d2ad75c6cd4 kind: Event lastTimestamp: "2026-07-07T02:49:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:25Z" name: success-200-isvc-4831c.18bfe209bb358c34 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: fb5eb018-07a9-4399-9ec0-83a49763e3a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "25503" uid: 04f29d90-a3c4-45e8-9c6a-5d2ad75c6cd4 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-4831c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4831c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-4831c.18bfe209bbb9903b namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: ac3b76bf-7481-45e5-b5cb-85dc770b47c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "25503" uid: 04f29d90-a3c4-45e8-9c6a-5d2ad75c6cd4 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4831c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-4831c.18bfe209bbb9ef6d namespace: kserve-ci-e2e-test resourceVersion: "25535" uid: e9a81353-d095-49f1-a725-d725139dceae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4831c namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 04f29d90-a3c4-45e8-9c6a-5d2ad75c6cd4 kind: Event lastTimestamp: "2026-07-07T02:49:25Z" message: InferenceService [success-200-isvc-4831c] is Ready metadata: creationTimestamp: "2026-07-07T02:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:25Z" name: success-200-isvc-4831c.18bfe216fbb7763b namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 101b3d11-0678-45e0-a388-3ed8735496f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:38:33.583446Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-524cd-predictor-b954c6b45-pqbdm to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f207230fd namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: e533b594-388a-4928-829e-040bfee3cec4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-524cd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f28036342 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 941bc766-28c8-4458-bd24-8363f2b741d8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f5ec8d2fd namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 0c9a8274-d90a-4ce0-9a6c-4d6d8f6e7a55 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f60476f80 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 5f2421fe-7b27-49a3-a96e-d8b46c67c3cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f6349c9ce namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: f4c48194-66e3-4593-a53b-4ba740d9c47c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f63adb568 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: 706fa861-3b19-41e5-8192-0f6d54d57fcf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f63b936eb namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 732b336f-209e-4ede-8a4f-cee4152289cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f66742803 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: d495e331-8bfe-453c-8ece-55cd56084f84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:38:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:34Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17f672b108f namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 6fa9cbf9-56c1-495f-8900-0955af16f88a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:39:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:12Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe17fb5e83685 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: bd2fd250-058e-4bcf-83f1-332d758be783 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe18f78c1f989 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: 805c6639-66a3-4717-b77e-cb555e2e32cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe18f78c2a054 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 26cac757-b339-40f3-8aa5-8d5ed439e4d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 12539b23-ceb1-41d4-97d9-732b3316d37e kind: Event lastTimestamp: "2026-07-07T02:39:47Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:47Z" name: success-200-isvc-524cd-predictor-b954c6b45-pqbdm.18bfe1903e4c5e4b namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: cd36413d-b252-4ded-aa36-fbdd8508c677 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-524cd-predictor-b954c6b45 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: b18354fe-911c-4308-95a3-35367d1d9876 kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'Created pod: success-200-isvc-524cd-predictor-b954c6b45-pqbdm' metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-524cd-predictor-b954c6b45.18bfe17f1fc701a5 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 481f4a1e-12a4-41fc-a1dd-1d209f613b96 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: dfb452b4-3ecc-40ab-8003-718442965bff kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Scaled up replica set success-200-isvc-524cd-predictor-b954c6b45 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-524cd-predictor.18bfe17f1d69247c namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: 440d5f40-2158-4556-84d7-be21d7e4958e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 8386903a-4926-4add-903a-5352bb22f0c7 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: success-200-isvc-524cd-predictor.18bfe1829c057a25 namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: d02810b0-5225-4da9-9f7e-81a622164f22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 8386903a-4926-4add-903a-5352bb22f0c7 kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: success-200-isvc-524cd-predictor.18bfe1829c69b25c namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 878f8a08-b4ce-4a35-8aa6-31cdb55427de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: 8386903a-4926-4add-903a-5352bb22f0c7 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:33Z" name: success-200-isvc-524cd-predictor.18bfe1899a401b9f namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 442d30a0-860a-4967-80ac-70ff08b3a193 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-524cd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: 8386903a-4926-4add-903a-5352bb22f0c7 kind: Event lastTimestamp: "2026-07-07T02:39:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:33Z" name: success-200-isvc-524cd-predictor.18bfe1899aa84367 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: 4872deab-39a9-4d54-9462-462389f6c262 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-524cd namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 8e0d227f-8a97-4fab-84a4-60bf9acee3f3 kind: Event lastTimestamp: "2026-07-07T02:39:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:22Z" name: success-200-isvc-524cd.18bfe17f1d803bf2 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: dc18e416-7eda-41da-8dda-9b4c27cec393 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-524cd namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: 8e0d227f-8a97-4fab-84a4-60bf9acee3f3 kind: Event lastTimestamp: "2026-07-07T02:39:22Z" message: InferenceService [success-200-isvc-524cd] is Ready metadata: creationTimestamp: "2026-07-07T02:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:22Z" name: success-200-isvc-524cd.18bfe18a7109716c namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 5f8b776b-d5cc-438d-b3c7-0a7870e31d6f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:39:43.866115Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7c238-predictor-54d4799ddc-wjsvv to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18f7d9e8f20 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 11073192-1102-4cf7-bfa7-3d61752cf1ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7c238-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18f88f3b2f2 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 01dd88ad-f5ce-49b6-ad5d-74b888422853 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:39:44Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fbb3e8668 namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 58c0a7a0-52ba-4d9c-a43c-d40bc5f23df1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fbcc2fafa namespace: kserve-ci-e2e-test resourceVersion: "21455" uid: 5427309c-b533-4532-a5c4-201a422c90a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fc0366408 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 5019653d-c3ad-42a8-bbc3-e6df91a7637a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fc0b17fd6 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: d8a6e0cc-faf9-4247-bd53-ffbd573799d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:44Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fc0bcbbf8 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: b1abea4f-3435-4cbf-890b-9a94c6d3efb7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:45Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fc3d71a27 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 85161860-d059-4696-a850-a008ecbd6e96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:39:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:45Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe18fc450ee8a namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: 4dbc8751-37e7-4423-9736-6c825b83a4cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:49:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:02Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe1901631990d namespace: kserve-ci-e2e-test resourceVersion: "26041" uid: 870252ca-8b2b-42aa-9650-af41a4eb5096 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe210b07c61a7 namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: d89aad2e-dc84-47d8-a2b7-02be57e7feb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe210b07da407 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 69f1cca0-e809-49e9-9ccf-b7b0204ae499 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e367f8d4-340b-4341-b0af-8df7a4f80c32 kind: Event lastTimestamp: "2026-07-07T02:49:02Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:02Z" name: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv.18bfe2118abe64d7 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: 7aeedddb-4283-4ba9-969d-ae22fc932b5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7c238-predictor-54d4799ddc namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: f18ba45d-cb75-4928-b62c-3120a82ddb62 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'Created pod: success-200-isvc-7c238-predictor-54d4799ddc-wjsvv' metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-7c238-predictor-54d4799ddc.18bfe18f7d0cbac3 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 99bd99dc-e0d9-4a17-9067-1363915450ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 21a94516-3ff4-4ef9-8acf-b121da66758a kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: Scaled up replica set success-200-isvc-7c238-predictor-54d4799ddc from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-7c238-predictor.18bfe18f7ade9ba0 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 44787f06-7e6f-4a2f-8d23-12ac9fae5ddb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: c1469a07-a4cf-4041-8822-6317e7ba14d1 kind: Event lastTimestamp: "2026-07-07T02:40:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:13Z" name: success-200-isvc-7c238-predictor.18bfe192f9c0fbe8 namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: e6956a74-4f4f-4cb1-b651-351b69d062fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: c1469a07-a4cf-4041-8822-6317e7ba14d1 kind: Event lastTimestamp: "2026-07-07T02:40:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:13Z" name: success-200-isvc-7c238-predictor.18bfe192fa23e5f8 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 6dcc0a03-e463-4a91-8fd6-2380519243cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: c1469a07-a4cf-4041-8822-6317e7ba14d1 kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:43Z" name: success-200-isvc-7c238-predictor.18bfe199f835571d namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: c9d089f1-f8fc-4c02-8ed7-b213f440f96e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:40:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c238-predictor namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: c1469a07-a4cf-4041-8822-6317e7ba14d1 kind: Event lastTimestamp: "2026-07-07T02:40:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:43Z" name: success-200-isvc-7c238-predictor.18bfe199f89ed50c namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: 20e72545-0e74-4448-aaf6-0b784cdbd749 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: aa59561f-f9cd-4a1f-8b08-3cf6eb117fd5 kind: Event lastTimestamp: "2026-07-07T02:40:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:32Z" name: success-200-isvc-7c238.18bfe18f7ae45a19 namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 5e43b049-782e-4dad-b346-fcd2457d235c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: aa59561f-f9cd-4a1f-8b08-3cf6eb117fd5 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-7c238": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7c238": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-7c238.18bfe18f7b8a15c9 namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 7e8ca814-2d37-4d32-946a-108db6231eb2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: aa59561f-f9cd-4a1f-8b08-3cf6eb117fd5 kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7c238": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-7c238.18bfe18f7b8a642f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 956b74fe-92b7-40fb-9ade-4a6d4046e270 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c238 namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: aa59561f-f9cd-4a1f-8b08-3cf6eb117fd5 kind: Event lastTimestamp: "2026-07-07T02:40:32Z" message: InferenceService [success-200-isvc-7c238] is Ready metadata: creationTimestamp: "2026-07-07T02:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:32Z" name: success-200-isvc-7c238.18bfe19ad168986f namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 0789b62d-f7fb-4c7d-a3ac-f131cdea93b9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:29:18.588570Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-821ea-predictor-6597d66d67-9vj2t to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:29:18Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fde82c6f8c namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: feeee1d8-017f-4cd2-af6e-739b289be0c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-821ea-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:18Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fdef10dc02 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: ce19df4e-b5a5-4d6b-950a-ad7e57090778 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe153a172c namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 786cc5ed-66e8-408a-8037-abb099a318de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe16e4bf93 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: fab05007-1cab-4cb7-8ab2-04567565ab3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe19c2302c namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: ed591bc6-a431-4c97-83ca-19f72ead21ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe1a333c6d namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: 6ee8ba7f-05f5-4e8a-8124-3670c3dd5df0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe1a3eb5ba namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 5718a5f3-bd41-484c-8e7d-34d2ca612430 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe1d410c13 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 4b6652fb-b677-4bbd-a359-3d7151e1bac5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:29:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:19Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe1db48494 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 2347c88f-3f67-4bb9-a356-477482a0b997 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:29:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:38:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:36Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe0fe619262fd namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 977422a4-cfe3-478d-b1a7-45eac2337da3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe17f1c068efb namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 5dbf601a-2b9d-4de7-8cc0-7140d60a7ae8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe17f1c071bae namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: d038577e-58eb-44f1-9fc2-9c416804d6e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 9c913aa5-b07f-433f-8f22-f15f154b5c35 kind: Event lastTimestamp: "2026-07-07T02:38:36Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:36Z" name: success-200-isvc-821ea-predictor-6597d66d67-9vj2t.18bfe17fd602ec9d namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: a74011a3-2c22-4638-85cc-39a0272d7685 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-821ea-predictor-6597d66d67 namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: c1805270-42b5-4b9c-953f-7b817f4a7aab kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: 'Created pod: success-200-isvc-821ea-predictor-6597d66d67-9vj2t' metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:18Z" name: success-200-isvc-821ea-predictor-6597d66d67.18bfe0fde7b3a4f3 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 8043b7bb-3b45-44d5-9d1b-c20a7343df21 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 462ab528-e144-478f-a9d4-8f8e9ccf483f kind: Event lastTimestamp: "2026-07-07T02:29:18Z" message: Scaled up replica set success-200-isvc-821ea-predictor-6597d66d67 from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:18Z" name: success-200-isvc-821ea-predictor.18bfe0fde574fd79 namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: 928f4984-d5c7-494d-bb40-899438ffd5e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:29:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 1f098b5c-63b2-4be2-b325-06e0f93c681c kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:48Z" name: success-200-isvc-821ea-predictor.18bfe101643bdd2e namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 1814871f-622a-4761-9344-9d8df37036d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:29:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 1f098b5c-63b2-4be2-b325-06e0f93c681c kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:48Z" name: success-200-isvc-821ea-predictor.18bfe1016494de32 namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 5af5e717-4a9c-45d6-9410-4a6ce4f05102 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:30:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 1f098b5c-63b2-4be2-b325-06e0f93c681c kind: Event lastTimestamp: "2026-07-07T02:30:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:33Z" name: success-200-isvc-821ea-predictor.18bfe10862804395 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 5b232e93-3a6b-4af7-957b-51f6dd6decfe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:30:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-821ea-predictor namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 1f098b5c-63b2-4be2-b325-06e0f93c681c kind: Event lastTimestamp: "2026-07-07T02:30:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:33Z" name: success-200-isvc-821ea-predictor.18bfe10862e07c2e namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: be13c1c4-a007-4fa0-bf45-69d7cebfa3eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:29:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-821ea namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 4336f364-faa5-4134-8d99-229af6c9c7a4 kind: Event lastTimestamp: "2026-07-07T02:30:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:30:06Z" name: success-200-isvc-821ea.18bfe0fde57a5799 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 6a29b355-626e-4047-bc39-52744c4d1f55 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-821ea namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 4336f364-faa5-4134-8d99-229af6c9c7a4 kind: Event lastTimestamp: "2026-07-07T02:30:06Z" message: InferenceService [success-200-isvc-821ea] is Ready metadata: creationTimestamp: "2026-07-07T02:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:30:06Z" name: success-200-isvc-821ea.18bfe109206c8e7d namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: efa2d06a-65da-417c-8453-8c448f0acbec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:49:49.311221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c74f5c63b namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 9186ba66-a84d-42c7-9466-cdab310a1c62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c8f9f1dd9 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: b5d9ac03-e1b5-4030-9115-e3551d8799ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c9122ca9e namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 8978b0f6-3a9e-4eb3-8a94-c452fb6a1ff7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c9451407a namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: b55ad97d-87c0-4f62-a9e4-020b8343dd72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c94effee2 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: fcd08630-f596-424b-a274-8203c28b2876 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c94fb5edd namespace: kserve-ci-e2e-test resourceVersion: "26616" uid: 243c5b5c-ea7b-4cc4-bd8e-63f443ec3dde reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c983a7c98 namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 66d1cd5b-d15a-4477-a66f-de967e2f9eed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21c98bee12b namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: f129cba2-425f-4b3a-a790-a2a723c7101c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:59:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:06Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe21ca30e2c2c namespace: kserve-ci-e2e-test resourceVersion: "30688" uid: 2ba3fb93-3631-495e-9103-6eae9a0df281 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:59:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:04Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe29da09e376e namespace: kserve-ci-e2e-test resourceVersion: "30653" uid: aed42dbb-d83e-4139-a296-982c53a6a578 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:59:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:04Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe29da0a62f8b namespace: kserve-ci-e2e-test resourceVersion: "30655" uid: 3ac6e9ff-430e-47f5-a60a-bed38b5ce7a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: d90e1d16-c558-4466-be42-16ab9fd74e88 kind: Event lastTimestamp: "2026-07-07T02:59:06Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:59:06Z" name: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6.18bfe29e17a35e4d namespace: kserve-ci-e2e-test resourceVersion: "30687" uid: 865ab8bf-8d73-46e1-9358-d0c9da33cc52 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8f974-predictor-6c87b89cbf namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 553c73c0-9ba2-4fdd-bf5c-f325ef57a1dd kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'Created pod: success-200-isvc-8f974-predictor-6c87b89cbf-rxkm6' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor-6c87b89cbf.18bfe21c748ae12d namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 682281b0-ab0d-487d-8d8b-999d1007a3ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: e02a389f-ea97-4551-80ca-9f496da827c8 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: Scaled up replica set success-200-isvc-8f974-predictor-6c87b89cbf from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974-predictor.18bfe21c72c42bb6 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: 802da438-1d0c-4af4-b259-978106854cb4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 0c1d9ee8-a6e7-4758-92dc-7e9b9843cd71 kind: Event lastTimestamp: "2026-07-07T02:50:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:19Z" name: success-200-isvc-8f974-predictor.18bfe21ff191c907 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 372f7e4d-c6e4-4b78-908a-c98247e3fac8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 0c1d9ee8-a6e7-4758-92dc-7e9b9843cd71 kind: Event lastTimestamp: "2026-07-07T02:50:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:19Z" name: success-200-isvc-8f974-predictor.18bfe21ff1f46f19 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 504e744e-2a23-427b-8fd3-b3643a8cd8e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:50:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 0c1d9ee8-a6e7-4758-92dc-7e9b9843cd71 kind: Event lastTimestamp: "2026-07-07T02:51:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:19Z" name: success-200-isvc-8f974-predictor.18bfe226ef668112 namespace: kserve-ci-e2e-test resourceVersion: "27410" uid: 4d350c84-5cb7-4e9b-9c50-b3078aa74742 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:50:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8f974-predictor namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 0c1d9ee8-a6e7-4758-92dc-7e9b9843cd71 kind: Event lastTimestamp: "2026-07-07T02:51:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:51:19Z" name: success-200-isvc-8f974-predictor.18bfe226efd2dae8 namespace: kserve-ci-e2e-test resourceVersion: "27412" uid: 00bea391-a92a-49c0-8c63-fef79a9c7007 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: cd775d50-d5f1-40d0-8fa5-e88a4e10ccb9 kind: Event lastTimestamp: "2026-07-07T02:50:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:50:36Z" name: success-200-isvc-8f974.18bfe21c72ff9df7 namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: f3c912a0-8b73-4d26-95b4-cebbfd2e5f08 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: cd775d50-d5f1-40d0-8fa5-e88a4e10ccb9 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-8f974": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8f974": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974.18bfe21c735e82e9 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 6bce8630-d44c-4805-92a7-dcb39d35b1dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: cd775d50-d5f1-40d0-8fa5-e88a4e10ccb9 kind: Event lastTimestamp: "2026-07-07T02:49:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8f974": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:49Z" name: success-200-isvc-8f974.18bfe21c735efbc2 namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: eecd6796-c363-4922-a9f8-5cd301bdb0f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8f974 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: cd775d50-d5f1-40d0-8fa5-e88a4e10ccb9 kind: Event lastTimestamp: "2026-07-07T02:50:36Z" message: InferenceService [success-200-isvc-8f974] is Ready metadata: creationTimestamp: "2026-07-07T02:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:50:36Z" name: success-200-isvc-8f974.18bfe2275e56b147 namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: 10f0d234-ba66-49bf-84e9-639cfdc3a50c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:48:58.827455Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2091-predictor-6c55945c5c-56pzp to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210b3e4b0e3 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: d6944433-2b61-4bb9-a8fc-8d76888b9b8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210ce2b2a92 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: f8c11110-3fb4-4b80-a1aa-e03adc8b6c68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210d09e9c9e namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 642b2ade-533d-4209-af7c-9e61edf274e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210d4b56731 namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: 3d78dab5-cc98-4bca-bb69-8d1e937d02f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210d5363f51 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 6e89595f-58c1-4fe0-a64d-372549b7e69e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210d542246e namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: 29eb4eb8-83fe-4e15-ae8a-826e40a00e4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210d98af623 namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 68b9b1e1-47f0-4236-b588-3d7d1c7c2d39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:48:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:59Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe210da129307 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 7c9918fb-7567-4549-a93d-8a5da3ac0b2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:49:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T02:49:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:36Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe2112d679958 namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: acd62eea-7afa-4032-90b3-4d0aec562fed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T03:06:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T03:06:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T03:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:18Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe302b28b7311 namespace: kserve-ci-e2e-test resourceVersion: "33653" uid: b9c396a0-6239-489f-ba5c-50cd0cc8f0ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T03:06:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T03:06:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T03:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:18Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe302b28c421b namespace: kserve-ci-e2e-test resourceVersion: "33654" uid: f003343b-6e3e-4159-991c-a98bf50d7d99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T03:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6781b79-010e-491b-ab3f-8786345e23b3 kind: Event lastTimestamp: "2026-07-07T03:06:21Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T03:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:21Z" name: success-200-isvc-b2091-predictor-6c55945c5c-56pzp.18bfe3038e089abd namespace: kserve-ci-e2e-test resourceVersion: "33730" uid: bae834b5-b917-4658-bb53-5a662f6894aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2091-predictor-6c55945c5c namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 7b5452fe-03dd-42d3-b25a-ee15fe5cc298 kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: 'Created pod: success-200-isvc-b2091-predictor-6c55945c5c-56pzp' metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-b2091-predictor-6c55945c5c.18bfe210b37511ba namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: ef9b4803-60b6-463c-b9c0-74762d186236 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 28f64f4d-558c-4687-8d3e-1494940b833f kind: Event lastTimestamp: "2026-07-07T02:48:58Z" message: Scaled up replica set success-200-isvc-b2091-predictor-6c55945c5c from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:48:58Z" name: success-200-isvc-b2091-predictor.18bfe210b159b316 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 0cc535e4-ae25-4454-93c7-2ee00e5f31e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "25906" uid: 262e40d3-7a16-4db7-a397-b107d6cc979a kind: Event lastTimestamp: "2026-07-07T02:49:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:28Z" name: success-200-isvc-b2091-predictor.18bfe2142fd25a49 namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: 6dcf113e-ba66-44fb-ab26-1f691f943878 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "25906" uid: 262e40d3-7a16-4db7-a397-b107d6cc979a kind: Event lastTimestamp: "2026-07-07T02:49:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:28Z" name: success-200-isvc-b2091-predictor.18bfe214302109d4 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: fcfcd508-baa8-4d96-9670-802a21a1472a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:49:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: 262e40d3-7a16-4db7-a397-b107d6cc979a kind: Event lastTimestamp: "2026-07-07T02:50:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:28Z" name: success-200-isvc-b2091-predictor.18bfe21b2e0f784f namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: 07315e54-5ddf-4136-ad4f-7ef44e33d28f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T02:49:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2091-predictor namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: 262e40d3-7a16-4db7-a397-b107d6cc979a kind: Event lastTimestamp: "2026-07-07T02:50:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:28Z" name: success-200-isvc-b2091-predictor.18bfe21b2e7b027c namespace: kserve-ci-e2e-test resourceVersion: "26990" uid: c44c2938-8df7-4a89-9bb2-44a3d77728e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T02:48:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2091 namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: ccf56472-1fcf-46b7-a647-4538ee281c59 kind: Event lastTimestamp: "2026-07-07T02:49:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:46Z" name: success-200-isvc-b2091.18bfe210b1911333 namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: e5890446-b590-479a-a78f-72f863e0e953 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: ccf56472-1fcf-46b7-a647-4538ee281c59 kind: Event lastTimestamp: "2026-07-07T02:49:46Z" message: InferenceService [success-200-isvc-b2091] is Ready metadata: creationTimestamp: "2026-07-07T02:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:46Z" name: success-200-isvc-b2091.18bfe21be872c470 namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: 05a9ce06-47a5-4261-8d87-258cb8efecb5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:38:03.624289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe17826be273f namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: f4de4bb5-bf1d-4fb0-809a-add52fa460a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-cec47-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe17833d58d9e namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 5f785048-10de-4b76-8aa3-1c165952397c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe178645ed635 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: d59186a8-324e-449a-aae8-923cbf072e49 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe17865c7c142 namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 31021723-f983-4801-b086-09ff4d86b96b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe17868bd16ff namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 09aaa027-2ee3-40f2-a775-ff08052d5461 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe1786937316f namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 9a75c27f-4927-4927-905c-bf9d241b19ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe17869b15292 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 4a5e696e-73e0-4cf1-b3b6-2edce9b9a78e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe1786d304fe4 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 61e82263-97d9-47d8-b903-35be6c6c8434 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:04Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe1786dfa4700 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 98a74912-f8ef-4d01-924f-959fc067cff9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:38:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:41Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe178b149511f namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 68d42bac-7e98-40f2-86cc-f2fc472f06bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe1888259f849 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: c5103459-1b04-412b-a752-3d9a24e5f7a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe188825ac615 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 76e24b4c-b12d-48d8-96c6-733ef91f085d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 42e934da-0017-49bf-b2af-90c5da5e4ff5 kind: Event lastTimestamp: "2026-07-07T02:39:17Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-07T02:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:17Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb.18bfe189752b4aff namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 3bc6fa78-3b1a-4973-b3a7-c418f6d4acc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cec47-predictor-58dfbcbf4f namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: daf6af97-06e4-4c84-8977-cfd438e1e905 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'Created pod: success-200-isvc-cec47-predictor-58dfbcbf4f-h6hkb' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-cec47-predictor-58dfbcbf4f.18bfe17826482e09 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 9690a10a-c7ac-44b3-8e62-eb65aa7aec85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: b5f3bbde-25aa-4a12-887d-3d9ea653bf41 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Scaled up replica set success-200-isvc-cec47-predictor-58dfbcbf4f from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-cec47-predictor.18bfe178224a3c6e namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: d32d4ff4-11ae-468d-8821-04d348b59609 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:38:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 6be56d92-46c0-4913-bf65-24f9244e0afc kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-cec47-predictor.18bfe17ba0d5544d namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 0dcb240f-7643-4faf-8335-64d568a8395a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:38:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 6be56d92-46c0-4913-bf65-24f9244e0afc kind: Event lastTimestamp: "2026-07-07T02:38:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:38:33Z" name: success-200-isvc-cec47-predictor.18bfe17ba1387f9b namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 6224b4f4-a8c4-417f-b71d-ad6af95bc8eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 6be56d92-46c0-4913-bf65-24f9244e0afc kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: success-200-isvc-cec47-predictor.18bfe1829ef0e26a namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 6b734793-ef68-4fc1-8a17-fc8f05325bcb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cec47-predictor namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 6be56d92-46c0-4913-bf65-24f9244e0afc kind: Event lastTimestamp: "2026-07-07T02:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:03Z" name: success-200-isvc-cec47-predictor.18bfe1829f49c919 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: b2df8070-d188-4665-a84e-9e19e9bfd381 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: b27623f5-ea7d-4429-ad8e-b626c8c274b8 kind: Event lastTimestamp: "2026-07-07T02:38:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:52Z" name: success-200-isvc-cec47.18bfe178225129ca namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 3810821c-157e-43cc-914e-eb63c589c887 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: b27623f5-ea7d-4429-ad8e-b626c8c274b8 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-cec47": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cec47": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-cec47.18bfe178230a18e5 namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 1b0c53df-30cf-4629-bd28-a120a481377e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: b27623f5-ea7d-4429-ad8e-b626c8c274b8 kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cec47": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:03Z" name: success-200-isvc-cec47.18bfe178230a820c namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 78ae568e-051b-4f49-90d1-e21be51d749b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cec47 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: b27623f5-ea7d-4429-ad8e-b626c8c274b8 kind: Event lastTimestamp: "2026-07-07T02:38:52Z" message: InferenceService [success-200-isvc-cec47] is Ready metadata: creationTimestamp: "2026-07-07T02:38:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:38:52Z" name: success-200-isvc-cec47.18bfe1836cf2d5d7 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: c1e68f04-95c7-432a-bb2e-cd12f80063fd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:39:13.928091Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf to ip-10-0-135-57.ec2.internal metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188852c93c8 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 978d460e-5cbb-4aff-8d20-aa330a7d4cce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e4a8c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:14Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe1888e5bc677 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 76262a86-6c5c-4021-b8e9-a3e637a855b7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:39:14Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188c3372149 namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 10dfc1cf-fa4e-4cc5-9837-5d75f4c4a637 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1702" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:14Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188c4c2b2f8 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 2b29249e-d90a-4d36-832d-5dc8e571c08e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188c7a95c7b namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: f940261a-045b-484b-abc6-a946fc93dc26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188c8254f70 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: b5ddd0b8-e31c-4266-913c-83b187d5f7a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188c831103e namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 40beafaa-e6f0-4827-9519-178e6375748e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188cb415eb1 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: e0a02f06-c1d0-44d3-b8b1-d7085fa0b93b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:39:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:39:15Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe188cbba68d2 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: d8fc4074-87d8-44d5-a1e0-e223598947f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T02:39:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:48:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:32Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe18911c9cda4 namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 3e3ed03a-c55b-43a3-9aa7-03394d4fc799 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe209ba5d240d namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: b204228f-c181-409e-a998-46505c069134 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:48:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T02:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:28Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe209ba5db6b8 namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 935735e1-9d9b-42ef-8d81-5ac08fb5f78f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:48:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: d21bec98-849a-41f5-8b49-622f9f6f0ea7 kind: Event lastTimestamp: "2026-07-07T02:48:32Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T02:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:48:32Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf.18bfe20a865c3c3b namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: 156f9abb-458c-4ae9-8f76-b0f0c73c2662 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-57.ec2.internal source: component: kubelet host: ip-10-0-135-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e4a8c-predictor-6c4869d48c namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: e55f1cf9-6c33-4fb7-829c-94df32317420 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: 'Created pod: success-200-isvc-e4a8c-predictor-6c4869d48c-wg5mf' metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-e4a8c-predictor-6c4869d48c.18bfe18884ce01ed namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: fcbee962-b5da-4f3d-8db7-0efeefca7774 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 96228d9c-3ddf-42ab-a8c2-5a8f85a8aab1 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: Scaled up replica set success-200-isvc-e4a8c-predictor-6c4869d48c from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-e4a8c-predictor.18bfe18882d77a63 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 25aca37a-7f54-4e6a-8c29-ee779578977b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 8ea88929-bd72-43de-84fa-a522fd31ed9c kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-e4a8c-predictor.18bfe18c01ceb3ec namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 1b5981b8-4fa6-47f3-b386-a21406f675eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:39:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 8ea88929-bd72-43de-84fa-a522fd31ed9c kind: Event lastTimestamp: "2026-07-07T02:39:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:39:43Z" name: success-200-isvc-e4a8c-predictor.18bfe18c0236881c namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 4aa657cb-79f3-43f7-824e-55e4c1c1d6fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 8ea88929-bd72-43de-84fa-a522fd31ed9c kind: Event lastTimestamp: "2026-07-07T02:40:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:28Z" name: success-200-isvc-e4a8c-predictor.18bfe192ffa5fb8c namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: 07a8efd4-fe41-42cf-b11d-57b8f26e8291 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T02:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 8ea88929-bd72-43de-84fa-a522fd31ed9c kind: Event lastTimestamp: "2026-07-07T02:40:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:40:28Z" name: success-200-isvc-e4a8c-predictor.18bfe193000b1960 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: e4106ea3-fb75-49be-b3eb-996ee2beefa0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 9862eade-0366-4e00-b747-32733ba0d4d5 kind: Event lastTimestamp: "2026-07-07T02:40:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:02Z" name: success-200-isvc-e4a8c.18bfe18882de94d7 namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: ab049e43-acbb-46e7-8479-b0fd1a1403d0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 9862eade-0366-4e00-b747-32733ba0d4d5 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-e4a8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e4a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-e4a8c.18bfe188834c92fd namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 551e9cfe-2979-4eb6-9e2b-8285f5b400b1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 9862eade-0366-4e00-b747-32733ba0d4d5 kind: Event lastTimestamp: "2026-07-07T02:39:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e4a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T02:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:39:13Z" name: success-200-isvc-e4a8c.18bfe188834ce01b namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 865322e8-9fb8-4aa5-b2f2-0aa3eee2cf80 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:40:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a8c namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 9862eade-0366-4e00-b747-32733ba0d4d5 kind: Event lastTimestamp: "2026-07-07T02:40:02Z" message: InferenceService [success-200-isvc-e4a8c] is Ready metadata: creationTimestamp: "2026-07-07T02:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:40:02Z" name: success-200-isvc-e4a8c.18bfe193cd4acc77 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: d47f5e19-d0c5-40ad-affd-c61d4d3a30d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:28:28.402022Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-2346e-7cbb7d79b-sfwft to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:28:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:28:28Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f238d27345 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 2c418f9e-060b-4a91-8ac8-855d195a37cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:28Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:28:28Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f251a29461 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 14e4e97a-42b6-4f98-a51b-46170d663b10 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2346e} kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" metadata: creationTimestamp: "2026-07-07T02:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:28Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f254473bfd namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: a4b74048-9ff1-4ecc-8b81-163147f6fe71 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2346e} kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:28:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" in 2.368s (2.368s including waiting). Image size: 218072665 bytes.' metadata: creationTimestamp: "2026-07-07T02:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:31Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f2e16d65f2 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 98745f78-c434-4d9b-ba46-720af5076b80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2346e} kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:28:31Z" message: 'Created container: switch-graph-2346e' metadata: creationTimestamp: "2026-07-07T02:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:31Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f2e4b96558 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: ed9d833a-36ad-43fa-b8cd-4e9840f09757 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2346e} kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:28:31Z" message: Started container switch-graph-2346e metadata: creationTimestamp: "2026-07-07T02:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:31Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f2e5293f29 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: f8fe5fb2-ce8e-4474-bc29-6338f692649e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2346e} kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:28:38Z" message: Stopping container switch-graph-2346e metadata: creationTimestamp: "2026-07-07T02:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:28:38Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f4911e872b namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 220d55ad-209e-4839-bb7c-0064928e6b74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-2346e} kind: Pod name: switch-graph-2346e-7cbb7d79b-sfwft namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 6eefb071-074a-49b8-a411-19b58c077bff kind: Event lastTimestamp: "2026-07-07T02:29:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:07Z" name: switch-graph-2346e-7cbb7d79b-sfwft.18bfe0f5966c192d namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: dc2626ed-2f5d-4529-9038-8cc1752ffa8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-2346e-7cbb7d79b namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: e89fca49-1815-46a2-b239-cad96801ae4b kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: 'Created pod: switch-graph-2346e-7cbb7d79b-sfwft' metadata: creationTimestamp: "2026-07-07T02:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:28Z" name: switch-graph-2346e-7cbb7d79b.18bfe0f237e0e310 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 9f441c18-60ba-47b1-bdd9-2df1680d6054 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-2346e namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 97aee985-9751-4084-8ecf-e8c74e2d55bf kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: Scaled up replica set switch-graph-2346e-7cbb7d79b from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:28:28Z" name: switch-graph-2346e.18bfe0f235dab43f namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 3e54e1cc-5bda-4609-afb4-4e6556cfac5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-2346e namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 38b802a6-a226-482b-8305-346e3d678ccb kind: Event lastTimestamp: "2026-07-07T02:28:28Z" message: InferenceGraph [switch-graph-2346e] is Ready metadata: creationTimestamp: "2026-07-07T02:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:28:28Z" name: switch-graph-2346e.18bfe0f240a1df7e namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 7f24317a-9a7d-4a36-bbc0-b2f3786f50d4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:29:48.775593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3ea7f-9cf7d776c-xhlql to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:29:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:29:48Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe104ef75d41f namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: e267a1c2-d907-4c76-9275-923d9dc6eac4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: "2026-07-07T02:29:49Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:29:49Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe105093f65b5 namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 0b7dd2e5-ead2-4ec7-b5a0-f47726c291c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3ea7f} kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: "2026-07-07T02:29:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:49Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe1050a85f5cc namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: ac77f3eb-2aa0-469e-9208-95a9eb196178 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3ea7f} kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: "2026-07-07T02:29:49Z" message: 'Created container: switch-graph-3ea7f' metadata: creationTimestamp: "2026-07-07T02:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:49Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe1050d979c0c namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 02325f11-7636-403e-93b3-e0c0c4bc9b14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3ea7f} kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: "2026-07-07T02:29:49Z" message: Started container switch-graph-3ea7f metadata: creationTimestamp: "2026-07-07T02:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:29:49Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe1050e07eb41 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: af1c82bc-e036-4f22-993d-f1747e787ec6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3ea7f} kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: "2026-07-07T02:38:03Z" message: Stopping container switch-graph-3ea7f metadata: creationTimestamp: "2026-07-07T02:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:03Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe178197aa735 namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 5d8be3bb-6c2c-42f0-8cf1-8c034e87a14b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T02:38:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3ea7f} kind: Pod name: switch-graph-3ea7f-9cf7d776c-xhlql namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 4665b529-1d20-4dd1-b943-69a2a65c200a kind: Event lastTimestamp: "2026-07-07T02:38:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T02:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:38:31Z" name: switch-graph-3ea7f-9cf7d776c-xhlql.18bfe178bd9cdcfd namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: dfdd68c9-5a7a-48b7-bacb-f8782c91959b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3ea7f-9cf7d776c namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 166d89ce-0e5a-492f-8e1f-02c063e92aeb kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: 'Created pod: switch-graph-3ea7f-9cf7d776c-xhlql' metadata: creationTimestamp: "2026-07-07T02:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:48Z" name: switch-graph-3ea7f-9cf7d776c.18bfe104eef8ccd8 namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 2a3ed257-a3a2-49dc-8ede-2a65c2f2d5bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: e653b603-dee7-4041-baec-309af3f30ef7 kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: Scaled up replica set switch-graph-3ea7f-9cf7d776c from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:29:48Z" name: switch-graph-3ea7f.18bfe104edaf3226 namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 24dcae31-11fe-43a2-8a22-9a2494665273 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:29:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 56ce235c-9a9d-4a84-aeeb-7fa9c374f9d8 kind: Event lastTimestamp: "2026-07-07T02:29:48Z" message: InferenceGraph [switch-graph-3ea7f] is Ready metadata: creationTimestamp: "2026-07-07T02:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:29:48Z" name: switch-graph-3ea7f.18bfe104f496ee9c namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: e6c308f4-77fe-482a-abc1-7e513792c7a3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:30:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: a5cbc57c-609c-46ab-8a71-5069837bdc97 kind: Event lastTimestamp: "2026-07-07T02:30:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:18Z" name: switch-graph-3ea7f.18bfe1086c34374e namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: b3feaede-c207-488a-9b4d-6e7047276bcc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T02:30:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3ea7f namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: a5cbc57c-609c-46ab-8a71-5069837bdc97 kind: Event lastTimestamp: "2026-07-07T02:30:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:30:18Z" name: switch-graph-3ea7f.18bfe1086ca44733 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 87ec660d-ae50-484e-b4e7-c9416c0a1bd5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T02:49:58.843597Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b2091-989b75cbf-g74p6 to ip-10-0-138-74.ec2.internal metadata: creationTimestamp: "2026-07-07T02:49:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T02:49:58Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe21ead223bc9 namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: 9d1d6106-4848-4960-9820-ed7acaf4a79f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df4cd5997-cljr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:59Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T02:49:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-b2091-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T02:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:59Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe21eb99864d5 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: bdabf564-0800-4dbe-bc3f-e628d196cf12 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:59Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T02:49:59Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T02:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T02:49:59Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe21eeaa141d9 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: b61fae99-9c90-434c-ae4e-12ab8868a7a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2091} kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T02:49:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d4cd17a4190a883a52dcd5ee86a7d271bf1d36bdd818eb1f96c33d5c7a9b76bf" already present on machine metadata: creationTimestamp: "2026-07-07T02:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:59Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe21eebffbd53 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 20ce272a-6441-4d8f-a45d-b0f617471798 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2091} kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T02:49:59Z" message: 'Created container: switch-graph-b2091' metadata: creationTimestamp: "2026-07-07T02:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:59Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe21eeef8427a namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: b00f8a26-e115-431a-86e1-8588da00e10d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2091} kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T02:49:59Z" message: Started container switch-graph-b2091 metadata: creationTimestamp: "2026-07-07T02:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T02:49:59Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe21eef6c641d namespace: kserve-ci-e2e-test resourceVersion: "26752" uid: be0ae43b-5520-438c-890c-387f0c1698e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T03:06:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2091} kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T03:06:18Z" message: Stopping container switch-graph-b2091 metadata: creationTimestamp: "2026-07-07T03:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:18Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe302a93110be namespace: kserve-ci-e2e-test resourceVersion: "33637" uid: 141794ac-4a2f-4c28-90a0-f21794c0f8e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T03:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b2091} kind: Pod name: switch-graph-b2091-989b75cbf-g74p6 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 378ba143-5cba-4943-afc1-b2d2a4f92f4a kind: Event lastTimestamp: "2026-07-07T03:06:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T03:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T03:06:41Z" name: switch-graph-b2091-989b75cbf-g74p6.18bfe303894af7ea namespace: kserve-ci-e2e-test resourceVersion: "33880" uid: 1f26f1f4-118f-4ef9-97b5-db456f8ba2d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-74.ec2.internal source: component: kubelet host: ip-10-0-138-74.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b2091-989b75cbf namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 323ab187-5117-47f6-b84d-8b2dd7cc6233 kind: Event lastTimestamp: "2026-07-07T02:49:58Z" message: 'Created pod: switch-graph-b2091-989b75cbf-g74p6' metadata: creationTimestamp: "2026-07-07T02:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:58Z" name: switch-graph-b2091-989b75cbf.18bfe21eac988ce4 namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: e3294937-27bc-41c9-942c-4675e19a7b64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: c9a61393-9e38-4e1e-8cd9-eaf8fae84614 kind: Event lastTimestamp: "2026-07-07T02:49:58Z" message: Scaled up replica set switch-graph-b2091-989b75cbf from 0 to 1 metadata: creationTimestamp: "2026-07-07T02:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:49:58Z" name: switch-graph-b2091.18bfe21eabe5535b namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: 3884c58f-cf45-4489-87f5-f72368e51760 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:49:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: 5f910830-dad7-42e9-a8de-8df96efc3597 kind: Event lastTimestamp: "2026-07-07T02:49:58Z" message: InferenceGraph [switch-graph-b2091] is Ready metadata: creationTimestamp: "2026-07-07T02:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T02:49:58Z" name: switch-graph-b2091.18bfe21eb1ad4b9e namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: b09055c1-ae09-42ae-9559-c5bddf6cad4d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: 844ce4ce-a5d3-42c7-9085-4420afe9b044 kind: Event lastTimestamp: "2026-07-07T02:50:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:13Z" name: switch-graph-b2091.18bfe2222a62d8c1 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: 21a502f3-c948-431d-a3e0-234e22f54e06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: 844ce4ce-a5d3-42c7-9085-4420afe9b044 kind: Event lastTimestamp: "2026-07-07T02:50:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T02:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:13Z" name: switch-graph-b2091.18bfe2222ac98995 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 964b9c23-9862-4846-b399-ba9f74c4a1a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: 844ce4ce-a5d3-42c7-9085-4420afe9b044 kind: Event lastTimestamp: "2026-07-07T02:50:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:28Z" name: switch-graph-b2091.18bfe225a9c69599 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: c51b4487-c298-4d52-bf83-e0b29e953a46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T02:50:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b2091 namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: 844ce4ce-a5d3-42c7-9085-4420afe9b044 kind: Event lastTimestamp: "2026-07-07T02:50:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T02:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T02:50:28Z" name: switch-graph-b2091.18bfe225aa2cd448 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: da579e3a-0f8b-47fb-8f3c-d9a42e5a34b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33911"