--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:18:59.504217Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-21436-665c75b9b7-kp5mh to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:18:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:18:59Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679e80e37448d namespace: kserve-ci-e2e-test resourceVersion: "22393" uid: 0af12141-b12d-4811-b2e4-e95d6677e965 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: "2026-09-18T17:19:00Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:19:00Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679e830af9bd0 namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: f32c8bbd-8bec-4f44-8165-bf98d97c1fe2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-21436} kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: "2026-09-18T17:19:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:00Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679e83214a4d5 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: f2025d54-e5a2-4585-8347-d2224def71af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-21436} kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: "2026-09-18T17:19:00Z" message: 'Created container: ensemble-graph-21436' metadata: creationTimestamp: "2026-09-18T17:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:00Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679e835396e1e namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: f8e4c6ab-3a9e-4c07-a161-eb341c7e9809 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-21436} kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: "2026-09-18T17:19:00Z" message: Started container ensemble-graph-21436 metadata: creationTimestamp: "2026-09-18T17:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:00Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679e836019430 namespace: kserve-ci-e2e-test resourceVersion: "22419" uid: 97a86326-2976-4451-a2a0-e573f6724121 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-21436} kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Stopping container ensemble-graph-21436 metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:10Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679ea8518dd79 namespace: kserve-ci-e2e-test resourceVersion: "22518" uid: 75e764c7-ed95-459a-a7dd-33fc872fac47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:19:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-21436} kind: Pod name: ensemble-graph-21436-665c75b9b7-kp5mh namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: db10332e-0ec3-4a3f-aab6-72be0c8234ae kind: Event lastTimestamp: "2026-09-18T17:19:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:37Z" name: ensemble-graph-21436-665c75b9b7-kp5mh.18d679eb01cd8ac2 namespace: kserve-ci-e2e-test resourceVersion: "22914" uid: aa7e0f27-91a2-4d57-afbe-c8ccd9a86c3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-21436-665c75b9b7 namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 5bad8368-a0b0-4266-970a-c9653278d5c1 kind: Event lastTimestamp: "2026-09-18T17:18:59Z" message: 'Created pod: ensemble-graph-21436-665c75b9b7-kp5mh' metadata: creationTimestamp: "2026-09-18T17:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:59Z" name: ensemble-graph-21436-665c75b9b7.18d679e80d5338e7 namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: 7ec4b396-7553-4690-9366-56507219f464 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-21436 namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: 3ba3369f-c193-43b5-94b1-b51ed8c94ecd kind: Event lastTimestamp: "2026-09-18T17:18:59Z" message: Scaled up replica set ensemble-graph-21436-665c75b9b7 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:59Z" name: ensemble-graph-21436.18d679e80a65df0c namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: 5a646a52-7038-4f50-b9d2-99ff684533fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-21436 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: c0bfe3f1-9717-45c7-99f7-9ec72b10908b kind: Event lastTimestamp: "2026-09-18T17:18:59Z" message: InferenceGraph [ensemble-graph-21436] is Ready metadata: creationTimestamp: "2026-09-18T17:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:59Z" name: ensemble-graph-21436.18d679e8131ddf67 namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: be3d2733-27dd-4aaa-b665-9a208d9ce311 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-21436 namespace: kserve-ci-e2e-test resourceVersion: "22377" uid: c0bfe3f1-9717-45c7-99f7-9ec72b10908b kind: Event lastTimestamp: "2026-09-18T17:18:59Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-21436": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-21436": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:59Z" name: ensemble-graph-21436.18d679e816784965 namespace: kserve-ci-e2e-test resourceVersion: "22404" uid: 075f888c-ffd9-4d99-9bd0-8b524fa7cd98 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-21436 namespace: kserve-ci-e2e-test resourceVersion: "22377" uid: c0bfe3f1-9717-45c7-99f7-9ec72b10908b kind: Event lastTimestamp: "2026-09-18T17:18:59Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-21436": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:59Z" name: ensemble-graph-21436.18d679e816789d67 namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: c373db1a-f179-4ae3-bcfb-e5c6c7cde0eb reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:20:20.349341Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23445" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-cca0b-846df546c8-8p7wd to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:20:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:20:20Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d679fae0f5e27a namespace: kserve-ci-e2e-test resourceVersion: "23451" uid: e32f43c0-2195-4b2f-be7d-2ab6738dc854 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:20Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:20:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-cca0b-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:20Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d679faeb5e3739 namespace: kserve-ci-e2e-test resourceVersion: "23462" uid: c1ac861f-f4df-4b5c-92de-e4d10c4f46e7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:21Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23454" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:20:21Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:20:21Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d679fb26ebbeff namespace: kserve-ci-e2e-test resourceVersion: "23472" uid: 5bc4890d-bf85-4846-b0f2-761e11c8aca5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cca0b} kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:20:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:21Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d679fb287a0e98 namespace: kserve-ci-e2e-test resourceVersion: "23474" uid: c69da964-1902-42d3-b215-e1195ecbd1f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cca0b} kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:20:21Z" message: 'Created container: ensemble-graph-cca0b' metadata: creationTimestamp: "2026-09-18T17:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:21Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d679fb2bf91253 namespace: kserve-ci-e2e-test resourceVersion: "23475" uid: 4a780b8f-3d43-47b7-bc0f-1c898c44f325 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cca0b} kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:20:21Z" message: Started container ensemble-graph-cca0b metadata: creationTimestamp: "2026-09-18T17:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:21Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d679fb2c7d9b4e namespace: kserve-ci-e2e-test resourceVersion: "23476" uid: e4049afc-da6d-4e97-b780-4b4859b467ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cca0b} kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Stopping container ensemble-graph-cca0b metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:35Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d67a6e2e6ebc7e namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: 8338a05f-222f-428f-895e-83224e01bd6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cca0b} kind: Pod name: ensemble-graph-cca0b-846df546c8-8p7wd namespace: kserve-ci-e2e-test resourceVersion: "23449" uid: 8b008697-9612-410f-83ee-93b5b25849a3 kind: Event lastTimestamp: "2026-09-18T17:29:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:03Z" name: ensemble-graph-cca0b-846df546c8-8p7wd.18d67a6edd25eab1 namespace: kserve-ci-e2e-test resourceVersion: "27588" uid: d9caf6c5-f408-4e8a-b253-7628f8304eca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-cca0b-846df546c8 namespace: kserve-ci-e2e-test resourceVersion: "23442" uid: b9bf43b6-7d9c-4b2f-9aef-e92cc9e1e7c9 kind: Event lastTimestamp: "2026-09-18T17:20:20Z" message: 'Created pod: ensemble-graph-cca0b-846df546c8-8p7wd' metadata: creationTimestamp: "2026-09-18T17:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:20Z" name: ensemble-graph-cca0b-846df546c8.18d679fae077947e namespace: kserve-ci-e2e-test resourceVersion: "23447" uid: cad2ebf9-0fc9-43e0-8587-b8c17184acba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-cca0b namespace: kserve-ci-e2e-test resourceVersion: "23440" uid: c9f05b25-f432-4153-a5c4-0b998ddb19bb kind: Event lastTimestamp: "2026-09-18T17:20:20Z" message: Scaled up replica set ensemble-graph-cca0b-846df546c8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:20Z" name: ensemble-graph-cca0b.18d679fadf7d24a4 namespace: kserve-ci-e2e-test resourceVersion: "23443" uid: c8b4ac65-7182-47a9-9c3e-bc431351cfed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-cca0b namespace: kserve-ci-e2e-test resourceVersion: "23458" uid: 8b889eae-951a-4ef6-9193-4cac028f5761 kind: Event lastTimestamp: "2026-09-18T17:20:20Z" message: InferenceGraph [ensemble-graph-cca0b] is Ready metadata: creationTimestamp: "2026-09-18T17:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:20Z" name: ensemble-graph-cca0b.18d679fae5267f2a namespace: kserve-ci-e2e-test resourceVersion: "23460" uid: 8bbd751b-ea12-4db4-a825-5f97383919d1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-cca0b namespace: kserve-ci-e2e-test resourceVersion: "23441" uid: 19f7ae3f-cb2a-402a-8958-81a1b54ecb43 kind: Event lastTimestamp: "2026-09-18T17:20:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:50Z" name: ensemble-graph-cca0b.18d679fe5e317af4 namespace: kserve-ci-e2e-test resourceVersion: "23721" uid: cf63d023-3dae-4afb-b7c2-6ea8f709385a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-cca0b namespace: kserve-ci-e2e-test resourceVersion: "23441" uid: 19f7ae3f-cb2a-402a-8958-81a1b54ecb43 kind: Event lastTimestamp: "2026-09-18T17:20:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:50Z" name: ensemble-graph-cca0b.18d679fe5e95711b namespace: kserve-ci-e2e-test resourceVersion: "23722" uid: c253b4c5-ed43-4104-abe1-430e66d08797 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:28:35.982729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27317" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:28:35Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e47033fa0 namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: 76c4657d-b7ee-42ee-8d03-ffb870956aaa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-054ab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:28:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:36Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e56902045 namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: 83716aa6-b20a-40c6-8ed9-e08887ccf4e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e8cf647d0 namespace: kserve-ci-e2e-test resourceVersion: "27353" uid: b1f01a7c-b6a9-489d-a34d-44f06960648f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e8e9cbd40 namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: 97caa622-dddd-4728-8384-1a7418adc145 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e91c6b405 namespace: kserve-ci-e2e-test resourceVersion: "27359" uid: badb824e-b32d-4f83-bfe8-ae68a5da238b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e924d1761 namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: b02304f5-c3bc-47bb-8d5a-eb0f7a731b7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e9259554b namespace: kserve-ci-e2e-test resourceVersion: "27362" uid: 822e25ef-4141-4403-9276-25584994cbae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e95296975 namespace: kserve-ci-e2e-test resourceVersion: "27364" uid: c0bb460b-4230-4f95-8800-0c458785a85e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6e95b016dc namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: dc2400bd-5ed2-454e-9820-edf4bfee64a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:29:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:24Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a6eeb4c683d namespace: kserve-ci-e2e-test resourceVersion: "27914" uid: 89fc5a25-6a02-42f5-84d4-43bfa7d57353 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:29:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:56Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a811fe676b6 namespace: kserve-ci-e2e-test resourceVersion: "28287" uid: df709a13-a9b0-4699-a8f1-eb0973976067 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:29:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:56Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a811fe70461 namespace: kserve-ci-e2e-test resourceVersion: "28288" uid: 543d9d8a-c3be-4260-9312-58f0dd047e56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: bd146766-527c-4b2e-ab51-c9a635f7af64 kind: Event lastTimestamp: "2026-09-18T17:29:59Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:59Z" name: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn.18d67a81c7a21cd2 namespace: kserve-ci-e2e-test resourceVersion: "28392" uid: 50a78941-4d88-4826-8291-203c59da15ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-054ab-predictor-9dbb95cd9 namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: 4826f58f-44ca-4570-9f9f-87e30de68d1c kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: 'Created pod: error-404-isvc-054ab-predictor-9dbb95cd9-dmxmn' metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:28:35Z" name: error-404-isvc-054ab-predictor-9dbb95cd9.18d67a6e462b4b7f namespace: kserve-ci-e2e-test resourceVersion: "27318" uid: 48aa1945-e291-4dab-98be-d57e21092347 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: fd9c32f2-be1f-4627-933e-8e311c8489cd kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Scaled up replica set error-404-isvc-054ab-predictor-9dbb95cd9 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:28:35Z" name: error-404-isvc-054ab-predictor.18d67a6e442bbdae namespace: kserve-ci-e2e-test resourceVersion: "27314" uid: ddd534e7-cb2a-4795-aa2a-6e091847960b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:28:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 6fe3942e-16a6-40f9-971b-9860039e22ee kind: Event lastTimestamp: "2026-09-18T17:29:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:20Z" name: error-404-isvc-054ab-predictor.18d67a71c251f099 namespace: kserve-ci-e2e-test resourceVersion: "27884" uid: 9e0dcc18-8328-4928-9c0c-0bbb66f4cbdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:28:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 6fe3942e-16a6-40f9-971b-9860039e22ee kind: Event lastTimestamp: "2026-09-18T17:29:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:21Z" name: error-404-isvc-054ab-predictor.18d67a71c2bf88c4 namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 29ec7bff-7340-47c8-bbee-78a991278fe1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27499" uid: 6fe3942e-16a6-40f9-971b-9860039e22ee kind: Event lastTimestamp: "2026-09-18T17:29:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:51Z" name: error-404-isvc-054ab-predictor.18d67a75429cb5fb namespace: kserve-ci-e2e-test resourceVersion: "28177" uid: f22fcec9-2145-491a-970f-033d011fe115 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27499" uid: 6fe3942e-16a6-40f9-971b-9860039e22ee kind: Event lastTimestamp: "2026-09-18T17:29:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:51Z" name: error-404-isvc-054ab-predictor.18d67a75430420f7 namespace: kserve-ci-e2e-test resourceVersion: "28178" uid: e6df63b9-d313-497a-8732-1418b573c35f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-054ab namespace: kserve-ci-e2e-test resourceVersion: "27294" uid: b17308e5-9034-450d-84fd-859d6b650d2a kind: Event lastTimestamp: "2026-09-18T17:29:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:29:34Z" name: error-404-isvc-054ab.18d67a6e43e2c1be namespace: kserve-ci-e2e-test resourceVersion: "28007" uid: 9d6039d6-e037-450f-86bf-0ebd011ac347 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-054ab namespace: kserve-ci-e2e-test resourceVersion: "28002" uid: b17308e5-9034-450d-84fd-859d6b650d2a kind: Event lastTimestamp: "2026-09-18T17:29:34Z" message: InferenceService [error-404-isvc-054ab] is Ready metadata: creationTimestamp: "2026-09-18T17:29:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:29:34Z" name: error-404-isvc-054ab.18d67a7bfc7abadf namespace: kserve-ci-e2e-test resourceVersion: "28004" uid: 25032f57-52ee-467a-9441-cd539ea0470e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:29:07.150021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27714" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0cb05-predictor-c8d4bc477-xvszg to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a7588ba6e17 namespace: kserve-ci-e2e-test resourceVersion: "27718" uid: b12a7eeb-3790-4c26-8956-2cc42ce11110 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27725" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75aac98c8b namespace: kserve-ci-e2e-test resourceVersion: "27736" uid: 798ac7de-1821-4b05-bcb1-711718894186 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75ac6278b0 namespace: kserve-ci-e2e-test resourceVersion: "27738" uid: d1100ec7-252f-4f2b-bd3e-5a5e405e37fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75af53ccc2 namespace: kserve-ci-e2e-test resourceVersion: "27740" uid: e19bbbff-33ca-4527-a0d0-85e7c360349f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75afd903c5 namespace: kserve-ci-e2e-test resourceVersion: "27741" uid: f3f3c68a-14f3-4ecd-80e4-16f7dba1a9b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75afe539a4 namespace: kserve-ci-e2e-test resourceVersion: "27742" uid: 8542de46-bea5-4523-988f-5e03b6e7cf5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75b45c18c4 namespace: kserve-ci-e2e-test resourceVersion: "27743" uid: 46f4dc2a-6146-4891-939c-f9d743254c6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75b4e52964 namespace: kserve-ci-e2e-test resourceVersion: "27744" uid: 5856f4b5-958a-4698-aef9-030b0880b2df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:29:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:54Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67a75ef94a635 namespace: kserve-ci-e2e-test resourceVersion: "28219" uid: e13fd82b-0ee2-4812-9de3-933e0f0e50c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:46:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:37Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67b6a0422fd8b namespace: kserve-ci-e2e-test resourceVersion: "35488" uid: 88a3577d-ea47-49ff-8546-43b2321ce1c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:46:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:37Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67b6a04238b34 namespace: kserve-ci-e2e-test resourceVersion: "35489" uid: 815070ed-672d-4d99-9e8c-363ee6bb72e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg namespace: kserve-ci-e2e-test resourceVersion: "27715" uid: 34b2cab2-d82a-48d1-9723-0f3d85195af8 kind: Event lastTimestamp: "2026-09-18T17:46:39Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:39Z" name: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg.18d67b6aa44c949f namespace: kserve-ci-e2e-test resourceVersion: "35542" uid: 06fbb923-20cf-4cba-a369-7d0bedeefbf2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0cb05-predictor-c8d4bc477 namespace: kserve-ci-e2e-test resourceVersion: "27707" uid: cedec7ab-2f20-43eb-9bb6-12e62e02b6a2 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: 'Created pod: error-404-isvc-0cb05-predictor-c8d4bc477-xvszg' metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor-c8d4bc477.18d67a758831bed2 namespace: kserve-ci-e2e-test resourceVersion: "27717" uid: a151f39f-c1bb-44e4-930f-ec6cfdd0f21f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27703" uid: 9fc38e8f-12b7-475e-b937-d6c281d09cb0 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: Scaled up replica set error-404-isvc-0cb05-predictor-c8d4bc477 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:07Z" name: error-404-isvc-0cb05-predictor.18d67a7583ec83ee namespace: kserve-ci-e2e-test resourceVersion: "27708" uid: 90d3c339-5c0e-4f5b-b5db-efd41334fbba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:29:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27706" uid: c4fe8d60-eaca-4368-a76a-0181517a79a6 kind: Event lastTimestamp: "2026-09-18T17:29:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:37Z" name: error-404-isvc-0cb05-predictor.18d67a79022bb771 namespace: kserve-ci-e2e-test resourceVersion: "28035" uid: 1a31834f-5307-4c11-9eff-98d8e2f1807e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:29:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27706" uid: c4fe8d60-eaca-4368-a76a-0181517a79a6 kind: Event lastTimestamp: "2026-09-18T17:29:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:37Z" name: error-404-isvc-0cb05-predictor.18d67a7902953c9c namespace: kserve-ci-e2e-test resourceVersion: "28036" uid: 173acf52-32d0-401c-89f8-97ff830af378 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27893" uid: c4fe8d60-eaca-4368-a76a-0181517a79a6 kind: Event lastTimestamp: "2026-09-18T17:30:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:22Z" name: error-404-isvc-0cb05-predictor.18d67a8000dad5c3 namespace: kserve-ci-e2e-test resourceVersion: "28631" uid: 1d2d773d-585c-40cf-998e-4d5831852e35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27893" uid: c4fe8d60-eaca-4368-a76a-0181517a79a6 kind: Event lastTimestamp: "2026-09-18T17:30:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:22Z" name: error-404-isvc-0cb05-predictor.18d67a8001c72a40 namespace: kserve-ci-e2e-test resourceVersion: "28632" uid: 3d74f2b1-9b6a-4e1a-bfc0-72644043398c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: e6f84e75-519d-4f77-a117-c17cdde62b23 kind: Event lastTimestamp: "2026-09-18T17:30:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:05Z" name: error-404-isvc-0cb05.18d67a7583fc9106 namespace: kserve-ci-e2e-test resourceVersion: "28455" uid: 20d64dc8-82d8-4753-9c97-f447532465bb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "28453" uid: e6f84e75-519d-4f77-a117-c17cdde62b23 kind: Event lastTimestamp: "2026-09-18T17:30:05Z" message: InferenceService [error-404-isvc-0cb05] is Ready metadata: creationTimestamp: "2026-09-18T17:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:05Z" name: error-404-isvc-0cb05.18d67a83045915e0 namespace: kserve-ci-e2e-test resourceVersion: "28454" uid: 7a66ac42-b17f-4de9-9c93-29ae3a0813be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:08:24.219094Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1ff44-predictor-66c97d8c59-4f788 to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d6795424458ac3 namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: d4f7fafb-a227-40de-b610-d56727d063a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d67954465740d7 namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 63ab95f7-2580-4175-acdc-e13cf6afae80 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679544896b79b namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 2e6a299d-cb3d-4466-9e9f-16c1f2fba4df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679544b6ae0e0 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: cf58fb8b-3fdc-4e6c-b0e8-ec36785ed858 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679544c0ee62e namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: df4e01a3-88c1-4195-a41b-6a1d52197bee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679544c1b3ef5 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 17effea0-aadf-4bbd-a3de-aa2835d53f9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679544ff976db namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 5dd178b6-1f93-47ec-932b-bd36ab73a0e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d67954507eef13 namespace: kserve-ci-e2e-test resourceVersion: "17217" uid: c325901e-68ae-4427-9695-9bb2ff301916 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:17:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:52Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d67954c7862ace namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 582b16a3-15ff-4684-b0c7-22810426e099 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:49Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679d7bde2bbf5 namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 17a50e81-06d3-4cfb-9aa7-5c15b19f4649 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:49Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679d7bde34a54 namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: 99067937-ffde-47b0-8b14-dbb7efc8c71a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: c3c8359d-7132-4959-b071-87f647ac270d kind: Event lastTimestamp: "2026-09-18T17:17:52Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:52Z" name: error-404-isvc-1ff44-predictor-66c97d8c59-4f788.18d679d8900334de namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 25e7b760-72fa-4d32-95c4-8ffabafbe86e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1ff44-predictor-66c97d8c59 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 6716ad37-1fe7-4b81-8707-89f30014ae65 kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: 'Created pod: error-404-isvc-1ff44-predictor-66c97d8c59-4f788' metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor-66c97d8c59.18d67954238afb71 namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: 27fa650a-c6f5-474e-b4c3-53e0cf3879da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 5f813dfd-f5f9-4efc-8e6c-f9f88b76394c kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Scaled up replica set error-404-isvc-1ff44-predictor-66c97d8c59 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44-predictor.18d679541e6191c3 namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: c8c2d173-acd9-47f5-91ea-edbad51ab7ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:08:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 982ef27d-2b4a-49fa-8005-5fc4ed1c067c kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:54Z" name: error-404-isvc-1ff44-predictor.18d679579d1f4477 namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: 177d3f6d-85d8-4d27-9b38-a9bd144100ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:08:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 982ef27d-2b4a-49fa-8005-5fc4ed1c067c kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:54Z" name: error-404-isvc-1ff44-predictor.18d679579d7ce9e1 namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: 3e098c96-a78d-4c46-9664-4a31265d7f2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 982ef27d-2b4a-49fa-8005-5fc4ed1c067c kind: Event lastTimestamp: "2026-09-18T17:09:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:54Z" name: error-404-isvc-1ff44-predictor.18d6795e9c2480de namespace: kserve-ci-e2e-test resourceVersion: "18232" uid: 03f33c60-8c67-400f-abb8-46c5ed0c6feb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 982ef27d-2b4a-49fa-8005-5fc4ed1c067c kind: Event lastTimestamp: "2026-09-18T17:09:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:54Z" name: error-404-isvc-1ff44-predictor.18d6795e9c9e5ef8 namespace: kserve-ci-e2e-test resourceVersion: "18233" uid: 66a17b61-ddad-4eb8-b32d-e25ab5ef45ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 7a6367fd-946a-4686-8421-ef948eefacfe kind: Event lastTimestamp: "2026-09-18T17:09:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:23Z" name: error-404-isvc-1ff44.18d679541f0ab0d7 namespace: kserve-ci-e2e-test resourceVersion: "17935" uid: 4aed9f61-6af0-40b3-bf5e-9d078e584252 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 7a6367fd-946a-4686-8421-ef948eefacfe kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-1ff44": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ff44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44.18d67954205fcef2 namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: 76af8278-0e0c-4cd0-a0b8-26fa3e7d66de reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 7a6367fd-946a-4686-8421-ef948eefacfe kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ff44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-1ff44.18d6795420603026 namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: f81ec552-35a0-4b8f-bd3b-74e5f59ea11a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "17932" uid: 7a6367fd-946a-4686-8421-ef948eefacfe kind: Event lastTimestamp: "2026-09-18T17:09:23Z" message: InferenceService [error-404-isvc-1ff44] is Ready metadata: creationTimestamp: "2026-09-18T17:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:23Z" name: error-404-isvc-1ff44.18d67961d767e57d namespace: kserve-ci-e2e-test resourceVersion: "17933" uid: 4ec98544-1e36-4f28-9fa8-937382a64489 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:17:49.551617Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:17:49Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7c4b742b6 namespace: kserve-ci-e2e-test resourceVersion: "21610" uid: e7819268-d43f-475a-9cfb-31e726e939dd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7e6bea88e namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: a5358c11-c413-49b0-a328-62aac9c05aa6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7e812e907 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: a86de73a-44ef-4bb8-9efc-d210b5ecb26b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7eac83242 namespace: kserve-ci-e2e-test resourceVersion: "21640" uid: 9a6ecba0-d6f1-4a48-99e7-cf29560108b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7eb4815a4 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 21bae94b-1e9c-400d-ad7c-5b035bfa65cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7eb53753c namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 12e3a858-963e-4b24-ba83-6195fcde3792 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7ee6e2971 namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: eee89b3d-cae8-45d5-a888-03857e55e9b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d7eeedf058 namespace: kserve-ci-e2e-test resourceVersion: "21645" uid: ef835372-f2d6-4d3a-919b-175d9e3a7eab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:18:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:38Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679d86164eb20 namespace: kserve-ci-e2e-test resourceVersion: "22190" uid: e86cb341-90a7-412c-b48a-be6b1d89ef2e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:10Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679ea9ed19b7a namespace: kserve-ci-e2e-test resourceVersion: "22586" uid: 6aa59dd7-4c5c-4915-9854-f5626b3eda24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:10Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679ea9ed24897 namespace: kserve-ci-e2e-test resourceVersion: "22587" uid: e2696042-9f1f-4437-bb25-74347ce97160 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 30a227ef-983f-44fe-95ec-94656ce728f5 kind: Event lastTimestamp: "2026-09-18T17:19:13Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:13Z" name: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8.18d679eb3db4ad8a namespace: kserve-ci-e2e-test resourceVersion: "22666" uid: ecc3133b-a908-4325-93bd-ce5553e04e9e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-21436-predictor-8574dbfbf7 namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 52f437c6-c04a-40f3-baab-6374568c73fd kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: 'Created pod: error-404-isvc-21436-predictor-8574dbfbf7-mdtp8' metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:17:49Z" name: error-404-isvc-21436-predictor-8574dbfbf7.18d679d7c43e4710 namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: 48347c44-05bd-4c5f-9028-0191241c955a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: b591f534-934e-4929-82a6-96197c57289d kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Scaled up replica set error-404-isvc-21436-predictor-8574dbfbf7 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:17:49Z" name: error-404-isvc-21436-predictor.18d679d7c27bd685 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: 642a032b-603c-406a-a69b-bc2789225d89 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:18:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: ceabfe55-2f6e-452d-981f-aace5ebf09cd kind: Event lastTimestamp: "2026-09-18T17:18:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:19Z" name: error-404-isvc-21436-predictor.18d679db408fe809 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 665561cd-0536-4bd9-b822-ba32c246c590 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:18:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: ceabfe55-2f6e-452d-981f-aace5ebf09cd kind: Event lastTimestamp: "2026-09-18T17:18:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:19Z" name: error-404-isvc-21436-predictor.18d679db40e566a2 namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 1093d1c9-1dd9-4565-9edb-c2f38f04a618 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:18:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: ceabfe55-2f6e-452d-981f-aace5ebf09cd kind: Event lastTimestamp: "2026-09-18T17:19:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:04Z" name: error-404-isvc-21436-predictor.18d679e23e4d73f2 namespace: kserve-ci-e2e-test resourceVersion: "22454" uid: 71f97ef6-174f-4f87-a7de-8b22f3061f3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:18:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: ceabfe55-2f6e-452d-981f-aace5ebf09cd kind: Event lastTimestamp: "2026-09-18T17:19:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:04Z" name: error-404-isvc-21436-predictor.18d679e23eb6cb68 namespace: kserve-ci-e2e-test resourceVersion: "22455" uid: 91649229-099f-4941-96ed-349cf48d38a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21436 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: 79590345-26e7-4af1-a491-bf57b8edfd3f kind: Event lastTimestamp: "2026-09-18T17:18:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:48Z" name: error-404-isvc-21436.18d679d7c213767f namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 270ce66a-4be1-446b-b52e-2f0711b1b684 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-21436 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: 79590345-26e7-4af1-a491-bf57b8edfd3f kind: Event lastTimestamp: "2026-09-18T17:18:48Z" message: InferenceService [error-404-isvc-21436] is Ready metadata: creationTimestamp: "2026-09-18T17:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:48Z" name: error-404-isvc-21436.18d679e572df7069 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: 47bc0386-3108-4532-9adf-80f3eed03f35 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:18:20.266138Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78e6b-predictor-547866bf4f-ss64v to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679deeb71a066 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 57b7d1f6-2005-4111-acc1-f84159299535 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df0def9651 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: 02d02c1b-da1f-4705-b023-130d900bfced reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df0f4a44ef namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: aa5d1b5e-f4ad-4644-a021-dccd0bd45c66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df122b04d0 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 82209e60-bf12-4900-a7e7-bda7f58a0d9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df12ad8d22 namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 0dc83555-c733-4b1f-a958-53d8083c5749 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df12b9acf2 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: 2eed598d-3cb7-4d99-91b7-f7a57e14aa37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df15dad848 namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: ab8118fb-1148-4785-8256-8f2f6df13c3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df16671cae namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: 76af81e6-c296-444a-8e77-fe5eca0ca51a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:19:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:07Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679df29644748 namespace: kserve-ci-e2e-test resourceVersion: "22484" uid: ad19165f-27cf-4120-b2d9-3979e8442922 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:41Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679f1c93c2825 namespace: kserve-ci-e2e-test resourceVersion: "23010" uid: 9fccc1f4-f20e-482d-b73f-7c7ebc81c87e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:41Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679f1c93c78f5 namespace: kserve-ci-e2e-test resourceVersion: "23012" uid: 67b43760-8a73-4055-ab6f-19a1f902c368 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 792a1fde-2c89-425b-83f2-42d73a9c9317 kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-78e6b-predictor-547866bf4f-ss64v.18d679f20586adaf namespace: kserve-ci-e2e-test resourceVersion: "23098" uid: 1bde3b5e-9348-4b4f-a685-b57cc420ba09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78e6b-predictor-547866bf4f namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 3682baf7-eb01-467f-a708-8119e226305f kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: 'Created pod: error-404-isvc-78e6b-predictor-547866bf4f-ss64v' metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor-547866bf4f.18d679deeafef0f0 namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 928b6f21-0412-44e2-a231-2ed1d25b2e91 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: 37e5b094-80b6-4f02-bd0e-5c8980247844 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Scaled up replica set error-404-isvc-78e6b-predictor-547866bf4f from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-78e6b-predictor.18d679dee8f66711 namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: e907ea62-bfe1-4374-bc4a-5f99d67fd95a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 9ed696eb-601b-47fd-9531-68f442c7ad95 kind: Event lastTimestamp: "2026-09-18T17:18:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:35Z" name: error-404-isvc-78e6b-predictor.18d679e26784295f namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: c7ecb1eb-6220-45ba-a517-5234ae2f5309 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 9ed696eb-601b-47fd-9531-68f442c7ad95 kind: Event lastTimestamp: "2026-09-18T17:18:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:35Z" name: error-404-isvc-78e6b-predictor.18d679e267d3c2e5 namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: 6b8608f7-8c29-473b-8341-7a0758026ed4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:18:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 9ed696eb-601b-47fd-9531-68f442c7ad95 kind: Event lastTimestamp: "2026-09-18T17:19:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:18:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:35Z" name: error-404-isvc-78e6b-predictor.18d679e5e68c5ed4 namespace: kserve-ci-e2e-test resourceVersion: "22898" uid: 19e33b6f-cf1e-4549-a261-74c5f8a49230 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:18:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 9ed696eb-601b-47fd-9531-68f442c7ad95 kind: Event lastTimestamp: "2026-09-18T17:19:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:18:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:35Z" name: error-404-isvc-78e6b-predictor.18d679e5e705953a namespace: kserve-ci-e2e-test resourceVersion: "22899" uid: 3da97588-4e74-4efc-baaa-8028bfd20069 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78e6b namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: a655c7fd-9829-4008-a447-82d87ca1bb17 kind: Event lastTimestamp: "2026-09-18T17:19:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:17Z" name: error-404-isvc-78e6b.18d679dee93953bf namespace: kserve-ci-e2e-test resourceVersion: "22716" uid: 43881f68-842e-4502-a65e-841ca273cde8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78e6b namespace: kserve-ci-e2e-test resourceVersion: "22714" uid: a655c7fd-9829-4008-a447-82d87ca1bb17 kind: Event lastTimestamp: "2026-09-18T17:19:17Z" message: InferenceService [error-404-isvc-78e6b] is Ready metadata: creationTimestamp: "2026-09-18T17:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:17Z" name: error-404-isvc-78e6b.18d679ec3d143b21 namespace: kserve-ci-e2e-test resourceVersion: "22715" uid: 878106ac-b806-4a71-ac9a-5c470dde9063 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:19:41.616195Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23057" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:19:41Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f1dc48ff2f namespace: kserve-ci-e2e-test resourceVersion: "23061" uid: 6d1576d2-5a5c-4652-bb33-bb371bf37d3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23065" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f1fe7b03cf namespace: kserve-ci-e2e-test resourceVersion: "23089" uid: 4e848147-b95d-422f-aa42-f78f76ecaf3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f2004d1c49 namespace: kserve-ci-e2e-test resourceVersion: "23091" uid: 64a8ceff-d76b-4ae8-8ae3-e0d7acb82c6f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f2038386ed namespace: kserve-ci-e2e-test resourceVersion: "23094" uid: 06c7f920-e909-4859-a847-a727a44436eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f2040461f8 namespace: kserve-ci-e2e-test resourceVersion: "23095" uid: b9ccecd7-6832-415d-a7b7-2891c339c904 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f2040ee640 namespace: kserve-ci-e2e-test resourceVersion: "23096" uid: cfea5d49-f02e-4b9f-865a-bee111e066da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f208049f25 namespace: kserve-ci-e2e-test resourceVersion: "23099" uid: c27ba64e-b1a3-46a4-8bf9-008dae65d6bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f20888fa79 namespace: kserve-ci-e2e-test resourceVersion: "23100" uid: 93e58ae2-5fca-465e-912d-e0668e4bfe46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:19:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:29:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:09Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d679f25619c212 namespace: kserve-ci-e2e-test resourceVersion: "27778" uid: 52db236f-9582-4320-92f0-489c40e2d37f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:06Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d67a75746c72b9 namespace: kserve-ci-e2e-test resourceVersion: "27672" uid: cb06c7de-9d58-4276-b6d3-27cfb4561727 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:06Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d67a75746cfe23 namespace: kserve-ci-e2e-test resourceVersion: "27674" uid: 42f360ad-4cb2-45c4-9587-07ef99d61d37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: bb9940db-4b7a-4e13-b987-1c560c5a458d kind: Event lastTimestamp: "2026-09-18T17:29:09Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:09Z" name: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9.18d67a761e88b584 namespace: kserve-ci-e2e-test resourceVersion: "27777" uid: cf1eb1e0-9f03-4dcb-bbee-23449316c325 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-792b5-predictor-7bddc6686c namespace: kserve-ci-e2e-test resourceVersion: "23051" uid: 3637185e-230f-4cce-bb74-61b5eaa23b87 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: 'Created pod: error-404-isvc-792b5-predictor-7bddc6686c-n5rw9' metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:41Z" name: error-404-isvc-792b5-predictor-7bddc6686c.18d679f1dbcdcd8b namespace: kserve-ci-e2e-test resourceVersion: "23058" uid: 9e4b47e3-4e10-4f74-9c74-1092e3b03014 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23049" uid: 7853b6f0-19fa-40ff-a72c-966da8c15c09 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Scaled up replica set error-404-isvc-792b5-predictor-7bddc6686c from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:41Z" name: error-404-isvc-792b5-predictor.18d679f1d9dfc0b6 namespace: kserve-ci-e2e-test resourceVersion: "23052" uid: 4528395f-a6fb-4f93-a63e-2dc30628cef5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 8a5c2c53-93aa-446f-823b-63d6207bb8aa kind: Event lastTimestamp: "2026-09-18T17:19:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:56Z" name: error-404-isvc-792b5-predictor.18d679f558592bd5 namespace: kserve-ci-e2e-test resourceVersion: "23229" uid: 3c1fcf5f-511f-44c4-9fc9-783c49abe7e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 8a5c2c53-93aa-446f-823b-63d6207bb8aa kind: Event lastTimestamp: "2026-09-18T17:19:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:56Z" name: error-404-isvc-792b5-predictor.18d679f558cd2fa5 namespace: kserve-ci-e2e-test resourceVersion: "23231" uid: 71b48d5a-158e-4ca7-aa47-aa04398ae224 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:20:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23230" uid: 8a5c2c53-93aa-446f-823b-63d6207bb8aa kind: Event lastTimestamp: "2026-09-18T17:20:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:56Z" name: error-404-isvc-792b5-predictor.18d679f8d809e945 namespace: kserve-ci-e2e-test resourceVersion: "23807" uid: 5b05d9e4-9962-40c8-9d54-02c95599d437 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:20:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23230" uid: 8a5c2c53-93aa-446f-823b-63d6207bb8aa kind: Event lastTimestamp: "2026-09-18T17:20:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:56Z" name: error-404-isvc-792b5-predictor.18d679f8d85df5ef namespace: kserve-ci-e2e-test resourceVersion: "23808" uid: 0da83b83-32ea-4a26-a075-fca12e719333 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23043" uid: 9e53e75d-fb74-4b9a-8c17-9c571452b199 kind: Event lastTimestamp: "2026-09-18T17:20:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:39Z" name: error-404-isvc-792b5.18d679f1d9ea101a namespace: kserve-ci-e2e-test resourceVersion: "23645" uid: 9e54c3c9-f323-499d-88b7-98ffc4648da8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23642" uid: 9e53e75d-fb74-4b9a-8c17-9c571452b199 kind: Event lastTimestamp: "2026-09-18T17:20:39Z" message: InferenceService [error-404-isvc-792b5] is Ready metadata: creationTimestamp: "2026-09-18T17:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:39Z" name: error-404-isvc-792b5.18d679ff65f5b5b3 namespace: kserve-ci-e2e-test resourceVersion: "23643" uid: a2a4b203-2905-4f7c-be6e-ac916df3604c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:08:54.673718Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17639" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9846a-predictor-797cb6675c-5kzcw to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:08:54Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b3b823d06 namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: bff53ef9-5de7-4111-8d6b-ba4669719103 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9846a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b42ff7b46 namespace: kserve-ci-e2e-test resourceVersion: "17655" uid: e14d2d37-0b33-4c0d-93be-0897e48425bf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b8147fa0a namespace: kserve-ci-e2e-test resourceVersion: "17683" uid: 54ce75b0-f6b0-483f-b425-ce8bfa81bd6c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b82cb2560 namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: e3001029-9ba5-42e8-84cd-9cfaaf4b4394 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b85aad790 namespace: kserve-ci-e2e-test resourceVersion: "17686" uid: 06bcd848-7f40-450e-9c29-160cb334a539 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b862348e2 namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: 7fe1119d-8cd9-499e-bc69-522b1378e198 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b862d9dcd namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 650006a8-2bf7-44be-87b7-fc4c51cbb817 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b891e7dc6 namespace: kserve-ci-e2e-test resourceVersion: "17690" uid: a37acbd1-0047-4b60-833b-aaa12b20a624 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795b8997a07e namespace: kserve-ci-e2e-test resourceVersion: "17691" uid: e3538824-37a5-4ab8-94bd-b8f88e7906d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:08:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:18:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:23Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d6795bcb4dd8b7 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: 2d6a86a1-4a8f-49ab-b81b-29c4c02064c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d679dedf0277fb namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: a6204abb-05b7-4085-9ce4-1f55b15acb23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d679dedf034d48 namespace: kserve-ci-e2e-test resourceVersion: "21947" uid: a9389b9f-93b7-4ffd-9a7a-57805706da9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw namespace: kserve-ci-e2e-test resourceVersion: "17641" uid: f8fc418d-fb75-41ce-a550-2e6ff248ac95 kind: Event lastTimestamp: "2026-09-18T17:18:23Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:23Z" name: error-404-isvc-9846a-predictor-797cb6675c-5kzcw.18d679df93b9b81d namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: 1c745836-d96e-4db7-927e-583a45a659a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9846a-predictor-797cb6675c namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 105566b0-dd3a-484f-a695-6ec692406697 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'Created pod: error-404-isvc-9846a-predictor-797cb6675c-5kzcw' metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:54Z" name: error-404-isvc-9846a-predictor-797cb6675c.18d6795b3ad5e3ec namespace: kserve-ci-e2e-test resourceVersion: "17640" uid: 17ca8ecc-b188-4819-9a72-27ebebddc368 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17628" uid: 91a9aef5-d4f6-46a4-96ba-8b39a35db5ae kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Scaled up replica set error-404-isvc-9846a-predictor-797cb6675c from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:54Z" name: error-404-isvc-9846a-predictor.18d6795b3841aedc namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: eb591040-609f-4313-a72e-e24d87ba29df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: d7f4f493-9abf-48dd-9fdb-1e710d98c638 kind: Event lastTimestamp: "2026-09-18T17:09:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:24Z" name: error-404-isvc-9846a-predictor.18d6795eb678c64c namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: 58ae270b-a605-47ef-b12d-34a87893aa76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: d7f4f493-9abf-48dd-9fdb-1e710d98c638 kind: Event lastTimestamp: "2026-09-18T17:09:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:24Z" name: error-404-isvc-9846a-predictor.18d6795eb6c83336 namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 0154141b-b607-4241-a21f-c78221021274 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:09:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17814" uid: d7f4f493-9abf-48dd-9fdb-1e710d98c638 kind: Event lastTimestamp: "2026-09-18T17:10:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:09Z" name: error-404-isvc-9846a-predictor.18d67965b4c23c93 namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: 5a5d9193-80d8-428a-9f14-5dc7c5dcf4d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:09:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17814" uid: d7f4f493-9abf-48dd-9fdb-1e710d98c638 kind: Event lastTimestamp: "2026-09-18T17:10:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:09Z" name: error-404-isvc-9846a-predictor.18d67965b54cb41b namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: 35dd25b4-862d-49bb-9020-24f22a724a3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9846a namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: 7ecb321e-10b1-4dfd-8da5-d4c08782c3ed kind: Event lastTimestamp: "2026-09-18T17:09:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:53Z" name: error-404-isvc-9846a.18d6795b380e741a namespace: kserve-ci-e2e-test resourceVersion: "18223" uid: 9a0de80d-9b83-43b1-acf2-ae1837c8e9d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9846a namespace: kserve-ci-e2e-test resourceVersion: "18221" uid: 7ecb321e-10b1-4dfd-8da5-d4c08782c3ed kind: Event lastTimestamp: "2026-09-18T17:09:53Z" message: InferenceService [error-404-isvc-9846a] is Ready metadata: creationTimestamp: "2026-09-18T17:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:53Z" name: error-404-isvc-9846a.18d67968dc26bf05 namespace: kserve-ci-e2e-test resourceVersion: "18222" uid: 71463a5f-9a6f-4985-84d9-a1dcb45a910e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:06:33.189640Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:06:33Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793a4a66c04f namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 4a525e78-e42a-4418-94ef-11bf0d13d040 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a4cbd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:33Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793a5869dbbf namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: da0765c9-5e37-4fcb-b4e9-1bcf7ad580df reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:35Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:06:35Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793ac67d0747 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: a4bdac90-9cba-4f4f-9d48-9c4392a6e69b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:35Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" metadata: creationTimestamp: "2026-09-18T17:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:35Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793ac80b44a9 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 02812aba-7136-4271-9b30-c98a774db35d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:47Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" in 12.362s (12.362s including waiting). Image size: 1385942088 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:47Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793da8eb21f4 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: d931bc90-3184-402d-965f-1635e8397930 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:47Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793dabe52aa9 namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 988e43ef-5279-4eb2-bd57-c94784520625 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:47Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793dac6e0a33 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: ffa8b176-ec10-4036-9c28-0afe2d4e4127 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:47Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:47Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793dac7dfdf3 namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 8a9661dd-b05b-4f6b-a602-b5f5b7c02993 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:50Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.298s (2.298s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:50Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793e358372b2 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 60d33a9f-26c6-4edb-9bba-158d9023cddd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:50Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793e38b1f859 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 659f94fa-e41f-40e1-a045-891c0d3ab806 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:06:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:50Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793e3931d9cc namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: b397ac35-ec93-4254-97a0-adb290ef2af8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:08:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:26Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6793e571e2b32 namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: 143291df-a580-4170-a6f6-781a98bebca5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6795419a9199f namespace: kserve-ci-e2e-test resourceVersion: "17156" uid: 922671b5-a855-4ce2-a7ea-030ab4e5ed59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d6795419a9ce24 namespace: kserve-ci-e2e-test resourceVersion: "17160" uid: 499b56dc-c974-481d-9c2e-ad1b563faffd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 32e51e91-1f04-4a84-b209-0682fd629c94 kind: Event lastTimestamp: "2026-09-18T17:08:26Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:26Z" name: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28.18d67954b160e23b namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: f20a440a-4bb3-4547-9023-f580a7dc8182 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a4cbd-predictor-7d7d594689 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: c74008ab-da19-44e6-8640-d88b13ac02c6 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: 'Created pod: error-404-isvc-a4cbd-predictor-7d7d594689-8gj28' metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:33Z" name: error-404-isvc-a4cbd-predictor-7d7d594689.18d6793a49e1bcf4 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 45872f11-4588-4246-b9aa-a37bade4506a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 74deab7b-7a41-4da6-806b-aa8f3110b638 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: Scaled up replica set error-404-isvc-a4cbd-predictor-7d7d594689 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:33Z" name: error-404-isvc-a4cbd-predictor.18d6793a480bf3c0 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: bcfeda8a-8ec7-4a83-bea2-39bd6bb79303 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:06:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 14729dea-9ad2-4452-b3fc-9d30afe04d7d kind: Event lastTimestamp: "2026-09-18T17:07:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:18Z" name: error-404-isvc-a4cbd-predictor.18d6793dc652f070 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: 21be4a38-7805-4670-be43-422ef2cff2d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:06:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 14729dea-9ad2-4452-b3fc-9d30afe04d7d kind: Event lastTimestamp: "2026-09-18T17:07:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:18Z" name: error-404-isvc-a4cbd-predictor.18d6793dc6d4e862 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 00805c3e-8587-495a-8f46-e4148055ab82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:07:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 14729dea-9ad2-4452-b3fc-9d30afe04d7d kind: Event lastTimestamp: "2026-09-18T17:08:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:18Z" name: error-404-isvc-a4cbd-predictor.18d6794843016d18 namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: b9b6c388-1099-467c-bccc-c0f99644941a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:07:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 14729dea-9ad2-4452-b3fc-9d30afe04d7d kind: Event lastTimestamp: "2026-09-18T17:08:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:18Z" name: error-404-isvc-a4cbd-predictor.18d67948437bf807 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 2b5fbf68-f8cb-4150-b7f0-33c5aefd55c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a4cbd namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: da8c7d41-55bc-4424-8a2a-b09f8a94bbbb kind: Event lastTimestamp: "2026-09-18T17:07:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:07:46Z" name: error-404-isvc-a4cbd.18d6793a47c77abb namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 7756752d-552a-439b-a0ff-f98f9079f348 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a4cbd namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: da8c7d41-55bc-4424-8a2a-b09f8a94bbbb kind: Event lastTimestamp: "2026-09-18T17:07:46Z" message: InferenceService [error-404-isvc-a4cbd] is Ready metadata: creationTimestamp: "2026-09-18T17:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:07:46Z" name: error-404-isvc-a4cbd.18d6794b6707bf0b namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: e8199419-801c-4f55-9ba3-01ed26b4852b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:19:10.688393Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22607" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:19:10Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679eaa8d841fe namespace: kserve-ci-e2e-test resourceVersion: "22611" uid: fa4afad8-9a6a-4659-a466-92fe377868b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22614" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679eacb0a6d3f namespace: kserve-ci-e2e-test resourceVersion: "22625" uid: 23332ad6-1020-4885-91c8-e07ef182dbc7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679eacd1ae9cd namespace: kserve-ci-e2e-test resourceVersion: "22627" uid: 8c5dd9e3-3445-4ec4-8152-665cf4090763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679ead02ae4b2 namespace: kserve-ci-e2e-test resourceVersion: "22629" uid: 08751de8-1e90-4957-8f38-ebc792a3af28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679ead0ae14ca namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: 8718af43-53e1-4d7e-ad64-f81837061f66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679ead0ba2dc5 namespace: kserve-ci-e2e-test resourceVersion: "22631" uid: beab241e-c9e8-44e3-8db9-44cf0151e533 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679ead3621617 namespace: kserve-ci-e2e-test resourceVersion: "22633" uid: 10249e17-8605-4bdb-96ba-cbdbcbfc41b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679ead3de2c9d namespace: kserve-ci-e2e-test resourceVersion: "22634" uid: 4382dc56-9f66-4c4e-8595-ac8d70978e49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:19:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:28:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:38Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d679eb16b209cf namespace: kserve-ci-e2e-test resourceVersion: "27385" uid: a8307186-6eb7-486c-895b-3ccc602ee826 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:35Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d67a6e39192ad9 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 25e44945-c660-46ef-9850-e9a5ee47094a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:35Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d67a6e3919e47c namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: 3c602cd3-96bc-4029-860c-493de3fb522f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: df76c216-e80b-41f0-bc1b-1384dc69cb83 kind: Event lastTimestamp: "2026-09-18T17:28:38Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:38Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh.18d67a6edf2fb00d namespace: kserve-ci-e2e-test resourceVersion: "27384" uid: 53d955c0-edd0-4a08-a3cc-8a5406137dde reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cca0b-predictor-6f7448c4bc namespace: kserve-ci-e2e-test resourceVersion: "22597" uid: 1f0e4111-082f-4abf-8f82-f90955029735 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: 'Created pod: error-404-isvc-cca0b-predictor-6f7448c4bc-mtmnh' metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:10Z" name: error-404-isvc-cca0b-predictor-6f7448c4bc.18d679eaa85a4f6b namespace: kserve-ci-e2e-test resourceVersion: "22612" uid: 089a4d4d-ae41-4c37-9376-9ee36dddae5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22595" uid: 313601f1-6af6-4707-994d-2bd15af9bfbc kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Scaled up replica set error-404-isvc-cca0b-predictor-6f7448c4bc from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:10Z" name: error-404-isvc-cca0b-predictor.18d679eaa3c88a1b namespace: kserve-ci-e2e-test resourceVersion: "22599" uid: dc892082-a991-4795-8b4f-9f945c182be0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22596" uid: a34c631c-7ba7-4d33-b65f-bafe09fba6d4 kind: Event lastTimestamp: "2026-09-18T17:19:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:40Z" name: error-404-isvc-cca0b-predictor.18d679ee223d01af namespace: kserve-ci-e2e-test resourceVersion: "22955" uid: 2162f23c-5c58-4267-ba09-39f1b09b1644 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22596" uid: a34c631c-7ba7-4d33-b65f-bafe09fba6d4 kind: Event lastTimestamp: "2026-09-18T17:19:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:40Z" name: error-404-isvc-cca0b-predictor.18d679ee22a77536 namespace: kserve-ci-e2e-test resourceVersion: "22956" uid: 1fd24a0c-0013-430c-b19b-39acdf2a42a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:19:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22789" uid: a34c631c-7ba7-4d33-b65f-bafe09fba6d4 kind: Event lastTimestamp: "2026-09-18T17:20:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:25Z" name: error-404-isvc-cca0b-predictor.18d679f520385f41 namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: 9d3f189e-b5c1-438c-98d4-4110ec13e588 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:19:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22789" uid: a34c631c-7ba7-4d33-b65f-bafe09fba6d4 kind: Event lastTimestamp: "2026-09-18T17:20:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:25Z" name: error-404-isvc-cca0b-predictor.18d679f520a66382 namespace: kserve-ci-e2e-test resourceVersion: "23509" uid: 5224cbbb-4e86-4382-86c4-cb7610860c95 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cca0b namespace: kserve-ci-e2e-test resourceVersion: "22584" uid: 2f0853fd-dc30-4686-ba76-ede6a82344d4 kind: Event lastTimestamp: "2026-09-18T17:20:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:08Z" name: error-404-isvc-cca0b.18d679eaa3c699b2 namespace: kserve-ci-e2e-test resourceVersion: "23333" uid: a170c73b-3607-45a0-b9bf-b05205fb3b9d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cca0b namespace: kserve-ci-e2e-test resourceVersion: "23331" uid: 2f0853fd-dc30-4686-ba76-ede6a82344d4 kind: Event lastTimestamp: "2026-09-18T17:20:08Z" message: InferenceService [error-404-isvc-cca0b] is Ready metadata: creationTimestamp: "2026-09-18T17:20:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:08Z" name: error-404-isvc-cca0b.18d679f827db4b76 namespace: kserve-ci-e2e-test resourceVersion: "23332" uid: 74ecaffd-cdb0-4aa9-ab85-2947e382d912 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:29:57.161557Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28330" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a812da5cf72 namespace: kserve-ci-e2e-test resourceVersion: "28337" uid: 34806286-4b93-474c-bd94-d524983f7d35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28340" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a814f6789fa namespace: kserve-ci-e2e-test resourceVersion: "28357" uid: 8c312e32-59d6-40ad-add3-208fca17d142 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a8150cc25bd namespace: kserve-ci-e2e-test resourceVersion: "28360" uid: 06588283-cb85-4d55-91ce-8e9f2bf75a55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a8153a9d8aa namespace: kserve-ci-e2e-test resourceVersion: "28363" uid: ef2d9fbf-f12b-43f1-b322-6184c75910c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a81543112ed namespace: kserve-ci-e2e-test resourceVersion: "28364" uid: a4426039-e6aa-4b2c-b908-5f1488ea4455 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a81543da82c namespace: kserve-ci-e2e-test resourceVersion: "28365" uid: 080750a8-c53e-4681-928b-f9817a16c7e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a815704c180 namespace: kserve-ci-e2e-test resourceVersion: "28366" uid: 3fda4d6f-1d5f-4344-8914-1afbd815bfb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a815786328a namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: bf846f20-45c5-4eb5-997b-633275c279b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:29:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:39:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:24Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67a81646b8420 namespace: kserve-ci-e2e-test resourceVersion: "32496" uid: 1de99d14-30aa-4def-a368-5d3681c4c50b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:39:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:22Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67b04b6860214 namespace: kserve-ci-e2e-test resourceVersion: "32477" uid: 3b9e9a51-44b0-4b3f-b5af-50619821192d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:39:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:22Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67b04b68699d0 namespace: kserve-ci-e2e-test resourceVersion: "32478" uid: 33598816-cba0-476c-ad6b-18711f17b1a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk namespace: kserve-ci-e2e-test resourceVersion: "28332" uid: 97b2eb5c-6ea0-42cb-a67e-38592004d3fc kind: Event lastTimestamp: "2026-09-18T17:39:24Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:24Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk.18d67b052cd234cf namespace: kserve-ci-e2e-test resourceVersion: "32495" uid: 8a38c46b-f003-40a5-bc47-4c34af4bd980 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f0fb0-predictor-668cf4bb89 namespace: kserve-ci-e2e-test resourceVersion: "28323" uid: a57cd9e2-afb7-4c36-b7ba-7d2d5a90667a kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: 'Created pod: error-404-isvc-f0fb0-predictor-668cf4bb89-qq7hk' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor-668cf4bb89.18d67a812cc01e4b namespace: kserve-ci-e2e-test resourceVersion: "28334" uid: 29dfb3ad-e1bd-41a9-8022-d1d85684907a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28320" uid: 5ade092c-45f4-4e16-8294-4c40eb44ab48 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Scaled up replica set error-404-isvc-f0fb0-predictor-668cf4bb89 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:57Z" name: error-404-isvc-f0fb0-predictor.18d67a812adc4b24 namespace: kserve-ci-e2e-test resourceVersion: "28325" uid: 9e908c8c-43be-4b77-9ba9-4e265de07038 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28322" uid: fb92ee14-6581-4193-8fb5-7adaae9afde9 kind: Event lastTimestamp: "2026-09-18T17:30:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:12Z" name: error-404-isvc-f0fb0-predictor.18d67a84a97885ad namespace: kserve-ci-e2e-test resourceVersion: "28520" uid: 6835e94a-b34a-4360-812c-e115ae750701 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28322" uid: fb92ee14-6581-4193-8fb5-7adaae9afde9 kind: Event lastTimestamp: "2026-09-18T17:30:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:12Z" name: error-404-isvc-f0fb0-predictor.18d67a84a9e2e040 namespace: kserve-ci-e2e-test resourceVersion: "28522" uid: 8d26e680-3f08-43ca-b39f-3fc3a2244f1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:30:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28521" uid: fb92ee14-6581-4193-8fb5-7adaae9afde9 kind: Event lastTimestamp: "2026-09-18T17:31:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:30:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:12Z" name: error-404-isvc-f0fb0-predictor.18d67a8828e4c319 namespace: kserve-ci-e2e-test resourceVersion: "29065" uid: 86185ff9-540f-4d00-b29d-6ad4261e6fc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:30:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28521" uid: fb92ee14-6581-4193-8fb5-7adaae9afde9 kind: Event lastTimestamp: "2026-09-18T17:31:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:30:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:12Z" name: error-404-isvc-f0fb0-predictor.18d67a8829613c38 namespace: kserve-ci-e2e-test resourceVersion: "29066" uid: df0a463e-ffd8-466d-ae18-147e04bae72e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28309" uid: 70d302bb-814f-45ee-b7e8-f32d8c3a3f64 kind: Event lastTimestamp: "2026-09-18T17:30:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:54Z" name: error-404-isvc-f0fb0.18d67a812ae8bcc4 namespace: kserve-ci-e2e-test resourceVersion: "28900" uid: 4f2ceff8-d984-4bc4-bdac-5a16048d54d8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28896" uid: 70d302bb-814f-45ee-b7e8-f32d8c3a3f64 kind: Event lastTimestamp: "2026-09-18T17:30:54Z" message: InferenceService [error-404-isvc-f0fb0] is Ready metadata: creationTimestamp: "2026-09-18T17:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:54Z" name: error-404-isvc-f0fb0.18d67a8e7bd59afe namespace: kserve-ci-e2e-test resourceVersion: "28898" uid: 70df3c23-3d7d-46e3-ac99-643adba36c34 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:06:32.691316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:06:32Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793a2cb2da66 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: d6472ca8-5152-446e-bc19-e32303c0e8c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793a4f041865 namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 95da1030-04dd-4ff1-ab66-7b80b4d2a338 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8a18d9d3be8c90736cc06c152435ddad34092af46716a1397ac9f058cd7b8474" metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793a51105402 namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: ea97ca45-c761-4c46-9ee9-8b82a8aff626 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:36Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8a18d9d3be8c90736cc06c152435ddad34092af46716a1397ac9f058cd7b8474" in 3.325s (3.325s including waiting). Image size: 308177834 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:36Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793b17408ce3 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 42002ef6-d52d-4d07-bbb9-02539cf90262 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:36Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793b1ab71a32 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 1d33c594-2fd2-4862-a5dc-c1b8dbe16443 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T17:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:36Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793b1b219d37 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 68df1396-882b-44f3-9aa7-b5cf76a14891 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:42Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1978" metadata: creationTimestamp: "2026-09-18T17:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:42Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793c57ba775b namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 765bee22-2b30-4348-bc8a-5d1c16673d43 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1978" in 11.667s (11.667s including waiting). Image size: 1604512144 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793f0f290e26 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 53b4519b-3a9d-4d41-9335-ada5a370bf87 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793f12ee9abc namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: d3d4b301-ae5e-450f-bec7-16dac450b475 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793f135a2949 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 71dec2f8-bb26-42d7-a1d1-055877a8dd1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793f136790dc namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 40b805aa-5938-474e-adf6-d692bdc65895 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.546s (2.546s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:56Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793fab2a887b namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 4bf0f297-9ac0-406b-808e-8ba61ec33964 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:56Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793fae5332c4 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: a7bd15ba-a200-4d63-992f-e9d9a18457a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:06:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:56Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793faebe4124 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 3c40bff6-6342-4877-af49-41c4549c9794 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T17:06:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:08:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:03Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6793fdbfe4e2d namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: 46fcd1b7-3723-4806-bba6-36cbad2760b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6795b2872cadf namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 9a7ca70f-a0fa-44fb-ba95-e1510da846d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6795b28753678 namespace: kserve-ci-e2e-test resourceVersion: "17587" uid: f3492933-433c-4c85-baf5-4eb371ddb80b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 2c3fccdf-e8bb-4978-bdfc-b39b5b4a01a1 kind: Event lastTimestamp: "2026-09-18T17:08:58Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:58Z" name: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6.18d6795c0899a6b9 namespace: kserve-ci-e2e-test resourceVersion: "17711" uid: a6948dff-9b2b-4691-8def-72e9eb9c92f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-759886b4dc namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 8de7be5c-c90e-4deb-aad8-6196e63a3be4 kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-759886b4dc-4k7z6' metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:32Z" name: isvc-sklearn-graph-1-predictor-759886b4dc.18d6793a2c288c2e namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: d7d509b9-602d-462d-ae61-969cfe709cbe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 3d01191a-a4f0-4a38-a58d-b95bcdb65aca kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-759886b4dc from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:32Z" name: isvc-sklearn-graph-1-predictor.18d6793a244032ad namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 3b5f03b4-52fa-4db7-bd60-778000fc5649 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: c5b7852a-5ff0-4901-8d6b-a2fd72dda9c5 kind: Event lastTimestamp: "2026-09-18T17:07:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:32Z" name: isvc-sklearn-graph-1-predictor.18d6793da96ab1d7 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: c3bd5d7c-d285-4308-9d8a-fa85a0f7022b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: c5b7852a-5ff0-4901-8d6b-a2fd72dda9c5 kind: Event lastTimestamp: "2026-09-18T17:07:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:32Z" name: isvc-sklearn-graph-1-predictor.18d6793da9f1473c namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: fd4a8ccd-ce0c-4510-95a4-edf9958b5dbd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:07:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: c5b7852a-5ff0-4901-8d6b-a2fd72dda9c5 kind: Event lastTimestamp: "2026-09-18T17:08:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:47Z" name: isvc-sklearn-graph-1-predictor.18d6794ba648f1bb namespace: kserve-ci-e2e-test resourceVersion: "17456" uid: ea6d62b5-53f6-4d18-b22a-f1f048dd5b9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:07:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: c5b7852a-5ff0-4901-8d6b-a2fd72dda9c5 kind: Event lastTimestamp: "2026-09-18T17:08:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:47Z" name: isvc-sklearn-graph-1-predictor.18d6794ba6bfe517 namespace: kserve-ci-e2e-test resourceVersion: "17457" uid: caa46e7f-2c5d-4e10-9053-e7d8ea28ff39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: d9662a48-e446-4dc6-b1e8-8ccafc19a1f7 kind: Event lastTimestamp: "2026-09-18T17:08:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:13Z" name: isvc-sklearn-graph-1.18d6793a2a01ce17 namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: f483e649-0b9e-4e26-8b2a-cbc1ee8e4071 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: d9662a48-e446-4dc6-b1e8-8ccafc19a1f7 kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:06:32Z" name: isvc-sklearn-graph-1.18d6793a2a6c552d namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 7d098310-7f76-438c-8122-aa408e8e242c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: d9662a48-e446-4dc6-b1e8-8ccafc19a1f7 kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:06:32Z" name: isvc-sklearn-graph-1.18d6793a2a6cad7b namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: db504f99-b381-49a1-8d8e-054c630e96a0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: d9662a48-e446-4dc6-b1e8-8ccafc19a1f7 kind: Event lastTimestamp: "2026-09-18T17:08:13Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-18T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:13Z" name: isvc-sklearn-graph-1.18d6795196c233d2 namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: f6125840-3e02-42c1-8a10-27747c6cd6c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:06:33.387988Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6793a56391568 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 9703e360-1525-4c4c-b5c8-263fbce989ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6793a782c422d namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: cf163ae8-dadf-4432-994f-7ef512541290 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8a18d9d3be8c90736cc06c152435ddad34092af46716a1397ac9f058cd7b8474" already present on machine metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6793a79b81fec namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 4660f6a5-f850-4a4a-9419-3ee54c560793 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:34Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6793a7dc0a192 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: aa3fdbf7-ad71-4d61-8e5e-af3540504259 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:34Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6793a7e3c3cb3 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 33983881-5df5-4a4d-a1fe-184b357afa02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:06:38Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1978" metadata: creationTimestamp: "2026-09-18T17:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:38Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6793b76876b89 namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 9ffcd055-38e2-4d45-a1be-a1f74cb99bc6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1978" in 26.003s (26.003s including waiting). Image size: 1604512144 bytes.' metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6794184766254 namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: fddd64b0-acc3-43d3-8d69-cd6e9e0eac66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d67941889caf0f namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: ed0acd5c-e650-4927-8103-9e064367571e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6794189ab052d namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: f6132c82-1e0e-4aaf-a48a-0c31c6eb094a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6794189b7b42e namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 9233b2ac-29b9-4424-82bf-86314f0496bd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.586s (2.586s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:06Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6794223e64445 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 0ccf9e9e-38c3-49e4-b8e5-98fad847aef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:06Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6794227530548 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 6dfd48c2-9848-4c0a-93a3-1c3dca973a3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:07:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:06Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6794227d33872 namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 74335e9e-83a9-475f-8f36-69aa2a103580 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T17:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d679427cdc9e6f namespace: kserve-ci-e2e-test resourceVersion: "17595" uid: da58e2a3-bac4-4da8-9d22-6d25c75d8f6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6795b24279998 namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: 27126e1f-631b-4cae-a96f-83fd851e6ee6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6795b2428133d namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: f358e01e-cc30-410d-9d7d-2ef8faf3a297 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 32d20160-2ce6-4e04-bc0a-16e4fb4e0fea kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'Readiness probe failed: Get "https://10.133.0.17:8643/healthz": dial tcp 10.133.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl.18d6795b2b70170a namespace: kserve-ci-e2e-test resourceVersion: "17594" uid: f6439110-eb9c-410f-9a61-8a2126566d75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5fc4c74ccb namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 2f1c48b9-8505-4fd6-a5cd-38df0513503e kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5fc4c74ccb-gfnfl' metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-2-predictor-5fc4c74ccb.18d6793a559522b2 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 16782085-f3ca-4adc-8584-20f81b7f0c34 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 68c267b4-b338-4ccc-81cf-f19a4c8e06b7 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5fc4c74ccb from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:33Z" name: isvc-sklearn-graph-2-predictor.18d6793a4edcb53f namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 876aaf3f-cb8b-4f7f-ac25-15397f89d415 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: d69e83fd-695e-487f-b532-f0ef40321277 kind: Event lastTimestamp: "2026-09-18T17:07:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:48Z" name: isvc-sklearn-graph-2-predictor.18d6793dcd4c18f1 namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: a458622a-9b63-4376-8e10-89b569b4cd71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: d69e83fd-695e-487f-b532-f0ef40321277 kind: Event lastTimestamp: "2026-09-18T17:07:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:48Z" name: isvc-sklearn-graph-2-predictor.18d6793dcdbd9544 namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: 79b4ce37-40a3-48f4-a3f8-4810d66dcebf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:07:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: d69e83fd-695e-487f-b532-f0ef40321277 kind: Event lastTimestamp: "2026-09-18T17:08:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:48Z" name: isvc-sklearn-graph-2-predictor.18d679484a08d176 namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: de6dc783-9a0d-45ed-85a8-8d33157af82f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:07:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: d69e83fd-695e-487f-b532-f0ef40321277 kind: Event lastTimestamp: "2026-09-18T17:08:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:48Z" name: isvc-sklearn-graph-2-predictor.18d679484a8841df namespace: kserve-ci-e2e-test resourceVersion: "17467" uid: e4a316ca-17ed-4f95-94a3-2ddf604d879e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: c5c17eef-42f0-4618-9a78-bfb2d8edae73 kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:24Z" name: isvc-sklearn-graph-2.18d6793a4eeab227 namespace: kserve-ci-e2e-test resourceVersion: "17208" uid: bb3c84fd-fc3a-4d58-bc8f-1da94fa0c1f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: c5c17eef-42f0-4618-9a78-bfb2d8edae73 kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:24Z" name: isvc-sklearn-graph-2.18d679543b67eecb namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: 22890743-a7a4-4aed-aef7-5d4e5e9f462e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:06:33.047173Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-kf9x5 to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:06:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793a41e8b002 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 738e157f-e218-4baa-b5b5-83fca81ee5f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793a4bcc2977 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: c66cb7db-4e95-4bba-a280-23ee7d61fec7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:06:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793a87b67039 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: bf1edde8-595d-4eea-823f-0512e1a371c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8a18d9d3be8c90736cc06c152435ddad34092af46716a1397ac9f058cd7b8474" already present on machine metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793a892a10b9 namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 2bcf8ac7-31d8-4aa5-959a-0909da15ee12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793a8c38b7ad namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 9d29280c-8df3-41d1-a3b8-b824a2b04984 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793a8cb00c8e namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 8a835e5d-e40f-4aaf-a58f-cb308ed61e43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:06:38Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-18T17:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6793b76771db5 namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 6e4d5c51-5869-40a6-90d7-7cc63bd70a0a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 26.649s (26.649s including waiting). Image size: 1436472580 bytes.' metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d67941aaea77da namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 9c18d223-a1fc-4f38-b49b-c8b02ab7cfd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d67941aed5b381 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: fdba898f-1655-4b25-b447-50ac5cde1135 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d67941af567376 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 987cd8fb-9bb0-4add-825b-dece7db903e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:04Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T17:07:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d67941af679387 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 8af193c6-716e-4be8-90e6-5a324afa8c5b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.962s (1.962s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d67942245c53da namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: c3f6efeb-9795-4ddb-aceb-1a003d6ee735 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6794227622f2c namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 21f66533-5428-4de6-a918-ab58ca5a9e6a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:07:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6794227ed79b9 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 07877708-7839-43b9-98ba-7497240a55b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:08:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d679427cdc5eb8 namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: b89c08b4-a8da-41ee-9c88-33ef901e32e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6795b2c9126d5 namespace: kserve-ci-e2e-test resourceVersion: "17597" uid: 1f548ca8-3e94-41b2-b3c9-d1d938a3baea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a194a4cd-c62b-4407-b1df-5e197e33ff49 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-kf9x5.18d6795b2c91a509 namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: 158797e9-d1f9-4c97-b71e-50e5d65f6f42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: cbd7d1b7-95c7-404d-8c0e-6693d5375259 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-kf9x5' metadata: creationTimestamp: "2026-09-18T17:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:33Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d6793a4124e995 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: b35010aa-c543-4b3d-90d2-cd54c7794ab3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: c6100745-4683-4a3c-b54e-404173216638 kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:32Z" name: isvc-xgboost-graph-predictor.18d6793a3d5fc97d namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: a9af8813-ea1e-40c2-87a4-49a59bfd9878 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 932c7e19-07b9-49b3-abff-d91bdf6efc44 kind: Event lastTimestamp: "2026-09-18T17:07:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:48Z" name: isvc-xgboost-graph-predictor.18d6793dbbe254c4 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 50dec5d2-a5f7-4f56-adf1-b99eea128189 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 932c7e19-07b9-49b3-abff-d91bdf6efc44 kind: Event lastTimestamp: "2026-09-18T17:07:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:48Z" name: isvc-xgboost-graph-predictor.18d6793dbc7bec22 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: b1968d60-3e3b-4734-8c33-2deaf2bc5bfb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:07:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 932c7e19-07b9-49b3-abff-d91bdf6efc44 kind: Event lastTimestamp: "2026-09-18T17:08:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:48Z" name: isvc-xgboost-graph-predictor.18d6794838b10ac9 namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: 1f895c8f-4f91-451e-bf49-8892c5da32c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:07:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 932c7e19-07b9-49b3-abff-d91bdf6efc44 kind: Event lastTimestamp: "2026-09-18T17:08:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:48Z" name: isvc-xgboost-graph-predictor.18d6794839239d09 namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: 65b42135-790b-411d-9abe-63fca51b5d83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 89bec7b6-1f4a-41ad-ae64-1d545ca43f10 kind: Event lastTimestamp: "2026-09-18T17:08:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:14Z" name: isvc-xgboost-graph.18d6793a3d55abc8 namespace: kserve-ci-e2e-test resourceVersion: "16992" uid: 854412ee-946b-42de-a340-4cc71728398b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 89bec7b6-1f4a-41ad-ae64-1d545ca43f10 kind: Event lastTimestamp: "2026-09-18T17:08:14Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-18T17:08:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:14Z" name: isvc-xgboost-graph.18d67951e1637af9 namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: ad27910a-132f-4089-87a2-a42318387c28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:08:42.572362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-dbc88f5f4-zzlmb to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:08:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:08:42Z" name: model-chainer-dbc88f5f4-zzlmb.18d679586a36329a namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: 48478dbe-4e0a-4c23-bd4c-35eb384a1054 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:42Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:08:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:42Z" name: model-chainer-dbc88f5f4-zzlmb.18d6795875657e70 namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: f4c60776-1d65-4d4e-8eb6-1af744e2c4e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:43Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17403" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:08:43Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:08:43Z" name: model-chainer-dbc88f5f4-zzlmb.18d67958b01d8a19 namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 6f12c44a-d3b1-4582-9e70-2527e9ec2fd2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:08:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:43Z" name: model-chainer-dbc88f5f4-zzlmb.18d67958b16cbd1f namespace: kserve-ci-e2e-test resourceVersion: "17419" uid: 3acbcd50-9922-455c-8b8f-d737cffd95c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:08:43Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-18T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:43Z" name: model-chainer-dbc88f5f4-zzlmb.18d67958b479897c namespace: kserve-ci-e2e-test resourceVersion: "17422" uid: 068603d5-e1e3-49e0-a598-576317697bd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:08:43Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-18T17:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:43Z" name: model-chainer-dbc88f5f4-zzlmb.18d67958b502cb4e namespace: kserve-ci-e2e-test resourceVersion: "17423" uid: 01609ccc-b998-46ad-95ae-d1b6e507e4e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:54Z" name: model-chainer-dbc88f5f4-zzlmb.18d6795b17c1820e namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: f9cd5882-9589-46a4-9360-42d547d4f657 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-dbc88f5f4-zzlmb namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: def449f5-3cc2-4fec-9278-291bebfd2b65 kind: Event lastTimestamp: "2026-09-18T17:09:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:09:20Z" name: model-chainer-dbc88f5f4-zzlmb.18d6795b764fd3d5 namespace: kserve-ci-e2e-test resourceVersion: "17899" uid: 970f851a-3000-46ca-9bbd-0932eeef11fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-dbc88f5f4 namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: ccaae603-7fca-497c-b7ef-b7790be4fd5f kind: Event lastTimestamp: "2026-09-18T17:08:42Z" message: 'Created pod: model-chainer-dbc88f5f4-zzlmb' metadata: creationTimestamp: "2026-09-18T17:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:42Z" name: model-chainer-dbc88f5f4.18d6795869b21a18 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: cb1b585f-0b7a-4825-ac9c-55af1a8d712b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 7630f806-6f7d-42c8-9732-b56785ab1913 kind: Event lastTimestamp: "2026-09-18T17:08:42Z" message: Scaled up replica set model-chainer-dbc88f5f4 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:42Z" name: model-chainer.18d6795868df88d4 namespace: kserve-ci-e2e-test resourceVersion: "17389" uid: a5104fce-9a2d-4a15-8f23-90a1a2012dbd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: 17c0de0c-355d-4b6f-9fe6-9b0e4bab1836 kind: Event lastTimestamp: "2026-09-18T17:08:42Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-18T17:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:42Z" name: model-chainer.18d679586eef7b3d namespace: kserve-ci-e2e-test resourceVersion: "17407" uid: f67d2f30-59aa-4ad7-bc91-f094aeb59529 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:19:30.151740Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22837" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-78e6b-76f9f95c9f-j8gxc to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679ef30f351e3 namespace: kserve-ci-e2e-test resourceVersion: "22842" uid: 7b3cd043-1e4b-49d3-b352-935bf8e9b5e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22849" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679ef5318618b namespace: kserve-ci-e2e-test resourceVersion: "22860" uid: d9457de3-e6be-4361-adcc-0433cff3547b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78e6b} kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679ef54c96f98 namespace: kserve-ci-e2e-test resourceVersion: "22862" uid: a838bfdf-f27c-4ef8-bdd2-e508c29f5858 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78e6b} kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: 'Created container: sequence-graph-78e6b' metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679ef57b73788 namespace: kserve-ci-e2e-test resourceVersion: "22863" uid: 88e0b05f-2147-4aa8-9f71-59bee6b5007d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78e6b} kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: Started container sequence-graph-78e6b metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679ef5839332b namespace: kserve-ci-e2e-test resourceVersion: "22864" uid: 970efb5e-706a-492d-af51-fccfd1c086d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78e6b} kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Stopping container sequence-graph-78e6b metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:41Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679f1c0df882f namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 2f5247a1-2689-42a9-9b49-261cbaf2761f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78e6b} kind: Pod name: sequence-graph-78e6b-76f9f95c9f-j8gxc namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 29750859-9558-4e4c-8f81-24100bdb0fdb kind: Event lastTimestamp: "2026-09-18T17:20:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:07Z" name: sequence-graph-78e6b-76f9f95c9f-j8gxc.18d679f2051a8911 namespace: kserve-ci-e2e-test resourceVersion: "23314" uid: be543d00-5de6-4e9c-9786-88de31c409a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-78e6b-76f9f95c9f namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: c2fd2e8b-d593-4cae-9a6a-fa6ea01834f8 kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: 'Created pod: sequence-graph-78e6b-76f9f95c9f-j8gxc' metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b-76f9f95c9f.18d679ef307b34b2 namespace: kserve-ci-e2e-test resourceVersion: "22840" uid: 0c64875c-be51-4df9-9ada-46d00dc70162 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-78e6b namespace: kserve-ci-e2e-test resourceVersion: "22832" uid: 8dd62ee8-a0e9-4546-af28-5626721da40f kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: Scaled up replica set sequence-graph-78e6b-76f9f95c9f from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b.18d679ef2f930e7d namespace: kserve-ci-e2e-test resourceVersion: "22835" uid: a3604005-c95d-4059-9369-ffd44d48da86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-78e6b namespace: kserve-ci-e2e-test resourceVersion: "22853" uid: eab6d682-8661-4a0b-ba0b-f264f6cbfe6e kind: Event lastTimestamp: "2026-09-18T17:19:30Z" message: InferenceGraph [sequence-graph-78e6b] is Ready metadata: creationTimestamp: "2026-09-18T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:30Z" name: sequence-graph-78e6b.18d679ef33e6a5a2 namespace: kserve-ci-e2e-test resourceVersion: "22855" uid: 766dd821-e850-4c2d-85a1-2e8cb1e81ac7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:20:51.515029Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23742" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-792b5-6df85cc95c-krkc5 to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:20:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:20:51Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a022294987d namespace: kserve-ci-e2e-test resourceVersion: "23747" uid: 100304da-7c06-4e8b-b858-9e6b3fc37f95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:20:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-792b5-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:51Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a023273d960 namespace: kserve-ci-e2e-test resourceVersion: "23758" uid: 67df07de-1024-48cb-81ae-f2f989f03b51 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23751" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:20:52Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:20:52Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a026881224c namespace: kserve-ci-e2e-test resourceVersion: "23766" uid: f35ed769-58bd-4a3c-943f-2b79dc82b40a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-792b5} kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:20:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:52Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a026ad9b0bc namespace: kserve-ci-e2e-test resourceVersion: "23768" uid: 5c1993fc-92c1-4b40-ba8e-d484701c021a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-792b5} kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:20:52Z" message: 'Created container: sequence-graph-792b5' metadata: creationTimestamp: "2026-09-18T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:52Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a026db222d6 namespace: kserve-ci-e2e-test resourceVersion: "23769" uid: 3eadc5d0-5489-4c19-8237-511d85427e89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-792b5} kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:20:52Z" message: Started container sequence-graph-792b5 metadata: creationTimestamp: "2026-09-18T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:20:52Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a026e360f1c namespace: kserve-ci-e2e-test resourceVersion: "23770" uid: 7db5ce09-ab83-48ec-be63-024b416e7401 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-792b5} kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: Stopping container sequence-graph-792b5 metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:06Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a75696f3bcf namespace: kserve-ci-e2e-test resourceVersion: "27636" uid: 5574bcc8-e066-4886-9ef5-2b1281dd40c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-792b5} kind: Pod name: sequence-graph-792b5-6df85cc95c-krkc5 namespace: kserve-ci-e2e-test resourceVersion: "23745" uid: 431670db-f2f5-4c41-a83f-6e51c6521a3e kind: Event lastTimestamp: "2026-09-18T17:29:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:34Z" name: sequence-graph-792b5-6df85cc95c-krkc5.18d67a761c22b602 namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: 36eb254e-66e0-41bc-914c-d27e4a38a7e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-792b5-6df85cc95c namespace: kserve-ci-e2e-test resourceVersion: "23739" uid: aad47269-d864-44ff-bd7b-8ed922c65506 kind: Event lastTimestamp: "2026-09-18T17:20:51Z" message: 'Created pod: sequence-graph-792b5-6df85cc95c-krkc5' metadata: creationTimestamp: "2026-09-18T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:51Z" name: sequence-graph-792b5-6df85cc95c.18d67a02221a6d95 namespace: kserve-ci-e2e-test resourceVersion: "23744" uid: 15978678-64dd-4152-a51f-436a7d1eca88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23737" uid: ae23b605-5d60-4936-8275-eee5a560d08b kind: Event lastTimestamp: "2026-09-18T17:20:51Z" message: Scaled up replica set sequence-graph-792b5-6df85cc95c from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:51Z" name: sequence-graph-792b5.18d67a0220d1b89f namespace: kserve-ci-e2e-test resourceVersion: "23740" uid: 41fd6dac-70b2-4aed-a8cc-a044f2918b1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23753" uid: 16ff7f9e-f867-4ccb-8148-efc89d6770de kind: Event lastTimestamp: "2026-09-18T17:20:51Z" message: InferenceGraph [sequence-graph-792b5] is Ready metadata: creationTimestamp: "2026-09-18T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:51Z" name: sequence-graph-792b5.18d67a0226cbcde2 namespace: kserve-ci-e2e-test resourceVersion: "23755" uid: 5181eb2f-a2ce-476c-b371-0482b3c35d72 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23738" uid: 3c6fee62-6886-4070-86ea-d9514a42500c kind: Event lastTimestamp: "2026-09-18T17:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:21:21Z" name: sequence-graph-792b5.18d67a059fa9f029 namespace: kserve-ci-e2e-test resourceVersion: "23989" uid: 03ea6b05-fd36-47ff-ad88-a174a70843a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23738" uid: 3c6fee62-6886-4070-86ea-d9514a42500c kind: Event lastTimestamp: "2026-09-18T17:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:21:21Z" name: sequence-graph-792b5.18d67a05a00f2257 namespace: kserve-ci-e2e-test resourceVersion: "23990" uid: ed17315f-9706-4363-8a7d-d8976b28ac1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23880" uid: 3c6fee62-6886-4070-86ea-d9514a42500c kind: Event lastTimestamp: "2026-09-18T17:21:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:21:36Z" name: sequence-graph-792b5.18d67a0c9e8767dc namespace: kserve-ci-e2e-test resourceVersion: "24090" uid: 11a806f6-f14c-4008-92cc-775273b02f09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23880" uid: 3c6fee62-6886-4070-86ea-d9514a42500c kind: Event lastTimestamp: "2026-09-18T17:21:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:21:36Z" name: sequence-graph-792b5.18d67a0c9efb0ded namespace: kserve-ci-e2e-test resourceVersion: "24091" uid: 6a42e378-19b9-4391-b8f7-640a54e89fcc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:10:04.402501Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18320" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9846a-6484c88c6c-6kbst to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d6796b77aaf9ed namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: 984dfcfb-818c-4c4e-9731-267d1dd3e1f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18329" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:10:04Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d6796b996fffaf namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: edf968f6-37f6-406c-a4b0-f3bbb89758e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9846a} kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:10:05Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" metadata: creationTimestamp: "2026-09-18T17:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:10:05Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d6796b9b9424c6 namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: caf916be-6864-46e8-bb78-32c0ef46b35d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9846a} kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:10:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" in 2.093s (2.093s including waiting). Image size: 211751504 bytes.' metadata: creationTimestamp: "2026-09-18T17:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:10:07Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d6796c186092c8 namespace: kserve-ci-e2e-test resourceVersion: "18363" uid: 5690df37-6387-45e7-9f90-232b866c56fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9846a} kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:10:07Z" message: 'Created container: sequence-graph-9846a' metadata: creationTimestamp: "2026-09-18T17:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:10:07Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d6796c1b45d5c6 namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: ca1787bb-6daa-46dd-969c-58777d56cd3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9846a} kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:10:07Z" message: Started container sequence-graph-9846a metadata: creationTimestamp: "2026-09-18T17:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:10:07Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d6796c1bbb1c99 namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: 4eccf8ef-8b44-4693-ae27-06b026d35ed4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9846a} kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:18:19Z" message: Stopping container sequence-graph-9846a metadata: creationTimestamp: "2026-09-18T17:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:19Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d679ded81053a7 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: ce6936af-30fb-447d-9788-f59818236557 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:18:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9846a} kind: Pod name: sequence-graph-9846a-6484c88c6c-6kbst namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: fd4cc05e-830f-46cf-8738-2610dbf8a1f1 kind: Event lastTimestamp: "2026-09-18T17:18:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:49Z" name: sequence-graph-9846a-6484c88c6c-6kbst.18d679dfd0d45c99 namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: 1c4cd777-ff00-491b-ba9e-6f5f8ec41c2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9846a-6484c88c6c namespace: kserve-ci-e2e-test resourceVersion: "18316" uid: 33dc015b-8f28-4f87-9509-c35b6a60a840 kind: Event lastTimestamp: "2026-09-18T17:10:04Z" message: 'Created pod: sequence-graph-9846a-6484c88c6c-6kbst' metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a-6484c88c6c.18d6796b76f63c46 namespace: kserve-ci-e2e-test resourceVersion: "18322" uid: 21ca2d82-549f-4bec-87cb-449ba3bcf26d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18314" uid: 5a08f70b-0a6f-4d7e-b8f7-c3cc86c07933 kind: Event lastTimestamp: "2026-09-18T17:10:04Z" message: Scaled up replica set sequence-graph-9846a-6484c88c6c from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a.18d6796b74a43e33 namespace: kserve-ci-e2e-test resourceVersion: "18317" uid: 9d9a3a37-b3b5-44a9-886d-23a98988a79b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: 741553d8-76c4-47aa-a9c4-5582c07d5dda kind: Event lastTimestamp: "2026-09-18T17:10:04Z" message: InferenceGraph [sequence-graph-9846a] is Ready metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a.18d6796b7cd66176 namespace: kserve-ci-e2e-test resourceVersion: "18334" uid: 1152245a-20c6-456d-9619-d4010b80944e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: 741553d8-76c4-47aa-a9c4-5582c07d5dda kind: Event lastTimestamp: "2026-09-18T17:10:04Z" message: 'Failed to update status for InferenceGraph "sequence-graph-9846a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-9846a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a.18d6796b80ca8763 namespace: kserve-ci-e2e-test resourceVersion: "18335" uid: 21a012ef-8061-4ce1-bb03-7dc4a8f89ea9 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: 741553d8-76c4-47aa-a9c4-5582c07d5dda kind: Event lastTimestamp: "2026-09-18T17:10:04Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-9846a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:10:04Z" name: sequence-graph-9846a.18d6796b80cae1a4 namespace: kserve-ci-e2e-test resourceVersion: "18337" uid: 1957e40a-4252-4828-8729-a101c187a3d8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18315" uid: 752ab864-a32c-4f6c-aac3-fe98bebb98ff kind: Event lastTimestamp: "2026-09-18T17:10:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:19Z" name: sequence-graph-9846a.18d6796ef35c78ad namespace: kserve-ci-e2e-test resourceVersion: "18461" uid: 97e62f13-62d1-4019-82e6-7bd6bdcccfed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18315" uid: 752ab864-a32c-4f6c-aac3-fe98bebb98ff kind: Event lastTimestamp: "2026-09-18T17:10:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:19Z" name: sequence-graph-9846a.18d6796ef409bd2e namespace: kserve-ci-e2e-test resourceVersion: "18463" uid: e78605b0-1dcb-4e8c-a69a-e3ad9386fb0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18460" uid: 752ab864-a32c-4f6c-aac3-fe98bebb98ff kind: Event lastTimestamp: "2026-09-18T17:10:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:10:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:34Z" name: sequence-graph-9846a.18d6797272d10f7d namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 0229ffa4-ca36-40cd-83bd-07b0b816df8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9846a namespace: kserve-ci-e2e-test resourceVersion: "18460" uid: 752ab864-a32c-4f6c-aac3-fe98bebb98ff kind: Event lastTimestamp: "2026-09-18T17:10:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:10:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:34Z" name: sequence-graph-9846a.18d67972735aba54 namespace: kserve-ci-e2e-test resourceVersion: "18571" uid: b3f2efe1-cbea-4073-8fcc-583b2793f9cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:29:45.844773Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28112" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-054ab-78f7d487d-zqdpd to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:29:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:29:45Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a7e8b1d6776 namespace: kserve-ci-e2e-test resourceVersion: "28117" uid: 7ef5b829-6546-4c80-8476-7c706e0a79fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:46Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28121" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: "2026-09-18T17:29:46Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:29:46Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a7ead58d11e namespace: kserve-ci-e2e-test resourceVersion: "28133" uid: e198c4f3-f765-463b-bc76-845b0158e4fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-054ab} kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28115" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: "2026-09-18T17:29:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:46Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a7eaeb748e3 namespace: kserve-ci-e2e-test resourceVersion: "28135" uid: b7ef913c-5a8a-4926-bc79-adc3072126b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-054ab} kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28115" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: "2026-09-18T17:29:46Z" message: 'Created container: splitter-graph-054ab' metadata: creationTimestamp: "2026-09-18T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:46Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a7eb2387718 namespace: kserve-ci-e2e-test resourceVersion: "28136" uid: 6cd3b891-fb1e-4057-a4df-3219bdd20cb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-054ab} kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28115" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: "2026-09-18T17:29:46Z" message: Started container splitter-graph-054ab metadata: creationTimestamp: "2026-09-18T17:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:46Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a7eb2b8a5da namespace: kserve-ci-e2e-test resourceVersion: "28137" uid: 771e8621-cc40-4834-9f20-ba3b300dd716 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-054ab} kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28115" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: "2026-09-18T17:29:56Z" message: Stopping container splitter-graph-054ab metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:56Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a81169f5863 namespace: kserve-ci-e2e-test resourceVersion: "28251" uid: e4ef7d30-0ff0-4858-8a1a-b783203ac14b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-054ab} kind: Pod name: splitter-graph-054ab-78f7d487d-zqdpd namespace: kserve-ci-e2e-test resourceVersion: "28115" uid: da82e16f-de62-41ee-be99-14405245f091 kind: Event lastTimestamp: "2026-09-18T17:30:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:30:22Z" name: splitter-graph-054ab-78f7d487d-zqdpd.18d67a814d4480bc namespace: kserve-ci-e2e-test resourceVersion: "28638" uid: b6ad1173-dd14-4032-a2a3-b0955891fcd0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-054ab-78f7d487d namespace: kserve-ci-e2e-test resourceVersion: "28109" uid: 5365db1e-00cb-4ba0-9c3e-62d4f89e7ad7 kind: Event lastTimestamp: "2026-09-18T17:29:45Z" message: 'Created pod: splitter-graph-054ab-78f7d487d-zqdpd' metadata: creationTimestamp: "2026-09-18T17:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:45Z" name: splitter-graph-054ab-78f7d487d.18d67a7e8a9951c6 namespace: kserve-ci-e2e-test resourceVersion: "28114" uid: 0e8c856f-78ac-4861-90a3-85a3d45ff1aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-054ab namespace: kserve-ci-e2e-test resourceVersion: "28107" uid: 86b51233-08eb-4e9d-a143-446bed218d35 kind: Event lastTimestamp: "2026-09-18T17:29:45Z" message: Scaled up replica set splitter-graph-054ab-78f7d487d from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:45Z" name: splitter-graph-054ab.18d67a7e897a03f6 namespace: kserve-ci-e2e-test resourceVersion: "28110" uid: 3287a420-9c96-472b-95d9-1323faacff00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-054ab namespace: kserve-ci-e2e-test resourceVersion: "28123" uid: 6e890961-7d8c-4b61-abb8-9bea799b6219 kind: Event lastTimestamp: "2026-09-18T17:29:45Z" message: InferenceGraph [splitter-graph-054ab] is Ready metadata: creationTimestamp: "2026-09-18T17:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:29:45Z" name: splitter-graph-054ab.18d67a7e8f24659c namespace: kserve-ci-e2e-test resourceVersion: "28125" uid: 8484611d-8f19-472f-a162-9fd5f3d30d04 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:31:07.049090Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29001" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-f0fb0-64569c6dcf-vjfj9 to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:31:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:31:07Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67a917344ef4d namespace: kserve-ci-e2e-test resourceVersion: "29008" uid: 3ef7f175-ba0a-4151-8530-eb767ddf5a37 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:07Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29005" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:31:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-f0fb0-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:31:07Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67a917d2dd11e namespace: kserve-ci-e2e-test resourceVersion: "29017" uid: e9c6ce9d-7601-4ffc-874d-db7d48383d69 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:08Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29011" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:31:08Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:31:08Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67a91b9117431 namespace: kserve-ci-e2e-test resourceVersion: "29028" uid: 3bf40041-b3f8-49a7-b167-f97464f06381 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f0fb0} kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29005" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:31:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:31:08Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67a91ba6788dc namespace: kserve-ci-e2e-test resourceVersion: "29030" uid: ae9f6c64-9ef2-49ab-9172-dbed954350c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f0fb0} kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29005" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:31:08Z" message: 'Created container: splitter-graph-f0fb0' metadata: creationTimestamp: "2026-09-18T17:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:31:08Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67a91bdd69a81 namespace: kserve-ci-e2e-test resourceVersion: "29032" uid: 325ce6c5-db32-4105-8e3b-5835bec31e3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f0fb0} kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29005" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:31:08Z" message: Started container splitter-graph-f0fb0 metadata: creationTimestamp: "2026-09-18T17:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:31:08Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67a91be54b300 namespace: kserve-ci-e2e-test resourceVersion: "29033" uid: 24de3843-bb29-490c-8b75-32e7a69be583 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f0fb0} kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29005" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:39:21Z" message: Stopping container splitter-graph-f0fb0 metadata: creationTimestamp: "2026-09-18T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:21Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67b04abbe862a namespace: kserve-ci-e2e-test resourceVersion: "32443" uid: 36cfb33c-1210-4693-b89f-a56d4ac807ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:39:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f0fb0} kind: Pod name: splitter-graph-f0fb0-64569c6dcf-vjfj9 namespace: kserve-ci-e2e-test resourceVersion: "29005" uid: 0cb3ae7c-aa0e-408f-a6e2-890dcaa839ff kind: Event lastTimestamp: "2026-09-18T17:39:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:50Z" name: splitter-graph-f0fb0-64569c6dcf-vjfj9.18d67b0564527112 namespace: kserve-ci-e2e-test resourceVersion: "32683" uid: 1dcebeff-0164-4d99-a3b4-a247ca209874 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-f0fb0-64569c6dcf namespace: kserve-ci-e2e-test resourceVersion: "28998" uid: 1fddecb4-311c-4009-bc27-7f87a071e29c kind: Event lastTimestamp: "2026-09-18T17:31:07Z" message: 'Created pod: splitter-graph-f0fb0-64569c6dcf-vjfj9' metadata: creationTimestamp: "2026-09-18T17:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:07Z" name: splitter-graph-f0fb0-64569c6dcf.18d67a9171ef4930 namespace: kserve-ci-e2e-test resourceVersion: "29002" uid: 8e71539e-7040-41c5-8219-0c14383876de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28996" uid: 2635135f-f9ab-4075-bebc-84f7bc6a5602 kind: Event lastTimestamp: "2026-09-18T17:31:07Z" message: Scaled up replica set splitter-graph-f0fb0-64569c6dcf from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:07Z" name: splitter-graph-f0fb0.18d67a9170cbe142 namespace: kserve-ci-e2e-test resourceVersion: "28999" uid: dff9db6f-34f2-4ded-9b4d-8b31020fcd4f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:31:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "29013" uid: ae90afdc-520b-41c8-a89d-7d00fba4dacb kind: Event lastTimestamp: "2026-09-18T17:31:07Z" message: InferenceGraph [splitter-graph-f0fb0] is Ready metadata: creationTimestamp: "2026-09-18T17:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:31:07Z" name: splitter-graph-f0fb0.18d67a917a864850 namespace: kserve-ci-e2e-test resourceVersion: "29014" uid: 29b0c8ca-f271-49a9-99ea-fcefaedecd1f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:31:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28997" uid: 9c26820d-0a2f-4f07-9d8d-8f73556bea06 kind: Event lastTimestamp: "2026-09-18T17:31:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:37Z" name: splitter-graph-f0fb0.18d67a94f0417b0e namespace: kserve-ci-e2e-test resourceVersion: "29252" uid: eed281fa-1637-492d-bc35-9b68bee98f78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:31:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28997" uid: 9c26820d-0a2f-4f07-9d8d-8f73556bea06 kind: Event lastTimestamp: "2026-09-18T17:31:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:37Z" name: splitter-graph-f0fb0.18d67a94f0aac76d namespace: kserve-ci-e2e-test resourceVersion: "29253" uid: 0d3ee193-366b-4eb1-b89c-9d2e796c64f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:28:35.902420Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:28:35Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e4239d06f namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: 4e84056b-cc1f-429c-9bbe-8e06f36311be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-054ab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:28:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:36Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e512785ef namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: ed848c02-a0f8-4cc4-93a8-256615a9eac3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27309" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e888f4787 namespace: kserve-ci-e2e-test resourceVersion: "27350" uid: 9852a6e2-f520-4075-a663-f4f3e668dbec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e8ad775c7 namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: 66883604-b376-4f12-bac2-7a2196ce9fbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e8e334443 namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 8e125793-0368-498f-9727-e9c51abf1c29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e8f43ce9f namespace: kserve-ci-e2e-test resourceVersion: "27357" uid: d3977651-c6b8-42fd-a803-3ff3e3ff3848 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e8f50356a namespace: kserve-ci-e2e-test resourceVersion: "27358" uid: 8480406d-0b6c-4630-9145-e0e3ca376c3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e92b52a5f namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: 3d4be46a-4c47-407a-8d2a-2c4571baa82a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:28:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:37Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6e9329c1eb namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: e085c1cf-cfc2-42de-b862-69542bf39aa1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:29:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:24Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a6edd660c85 namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: c91161b4-6efd-4dd2-a1f1-889c3fa1b1aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:29:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:56Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a811d0e3722 namespace: kserve-ci-e2e-test resourceVersion: "28270" uid: 48ae4a45-01ee-4574-b1f9-2e1a4b452cf6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:29:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:56Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a811d0fdda9 namespace: kserve-ci-e2e-test resourceVersion: "28272" uid: 448d2dc6-21de-4de9-a856-928ffb25d7e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a6d09c8f-128f-4aa7-846e-dde485898f05 kind: Event lastTimestamp: "2026-09-18T17:29:59Z" message: 'Readiness probe failed: Get "https://10.132.0.45:8643/healthz": dial tcp 10.132.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:59Z" name: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7.18d67a81b9d04661 namespace: kserve-ci-e2e-test resourceVersion: "28391" uid: 61ab5dca-babb-4d75-9b89-2484b2e097e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-054ab-predictor-665d69d7f6 namespace: kserve-ci-e2e-test resourceVersion: "27288" uid: 64f1f28a-9208-4f7c-a8f3-561cacbc0b52 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: 'Created pod: success-200-isvc-054ab-predictor-665d69d7f6-pdpl7' metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:28:35Z" name: success-200-isvc-054ab-predictor-665d69d7f6.18d67a6e41a91544 namespace: kserve-ci-e2e-test resourceVersion: "27297" uid: c495dc1a-0873-48bb-a105-4c47b319760c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: 699dd0e4-2d5b-471a-a58a-e76a159860c9 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Scaled up replica set success-200-isvc-054ab-predictor-665d69d7f6 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:28:35Z" name: success-200-isvc-054ab-predictor.18d67a6e3d7be52a namespace: kserve-ci-e2e-test resourceVersion: "27290" uid: d1e425a8-9b1f-4444-8ffb-c0594d888c0c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:28:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27287" uid: f9b0da43-9c1a-4cf1-b4c5-4f7bb50b272c kind: Event lastTimestamp: "2026-09-18T17:29:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:20Z" name: success-200-isvc-054ab-predictor.18d67a71bbf260f7 namespace: kserve-ci-e2e-test resourceVersion: "27880" uid: 0146c73a-4532-43c8-9fa0-5e118117c7be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:28:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27287" uid: f9b0da43-9c1a-4cf1-b4c5-4f7bb50b272c kind: Event lastTimestamp: "2026-09-18T17:29:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:20Z" name: success-200-isvc-054ab-predictor.18d67a71bc54268e namespace: kserve-ci-e2e-test resourceVersion: "27881" uid: 4c13ec03-4660-4563-b931-bc3996c294ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27495" uid: f9b0da43-9c1a-4cf1-b4c5-4f7bb50b272c kind: Event lastTimestamp: "2026-09-18T17:29:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:50Z" name: success-200-isvc-054ab-predictor.18d67a753b12e33e namespace: kserve-ci-e2e-test resourceVersion: "28172" uid: 9809f218-4519-4d94-8fea-e41c223b64ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-054ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "27495" uid: f9b0da43-9c1a-4cf1-b4c5-4f7bb50b272c kind: Event lastTimestamp: "2026-09-18T17:29:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:50Z" name: success-200-isvc-054ab-predictor.18d67a753b7ca991 namespace: kserve-ci-e2e-test resourceVersion: "28173" uid: fe7fdb9a-76d8-418c-8d1f-5fc7fa2f4920 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-054ab namespace: kserve-ci-e2e-test resourceVersion: "27276" uid: 62190372-fddd-459a-ae6a-ed2e07e4cd50 kind: Event lastTimestamp: "2026-09-18T17:29:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:29:34Z" name: success-200-isvc-054ab.18d67a6e3d779c28 namespace: kserve-ci-e2e-test resourceVersion: "27996" uid: 9b72d410-d709-434b-bf5e-74e935516719 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-054ab namespace: kserve-ci-e2e-test resourceVersion: "27992" uid: 62190372-fddd-459a-ae6a-ed2e07e4cd50 kind: Event lastTimestamp: "2026-09-18T17:29:34Z" message: InferenceService [success-200-isvc-054ab] is Ready metadata: creationTimestamp: "2026-09-18T17:29:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:29:34Z" name: success-200-isvc-054ab.18d67a7bee270694 namespace: kserve-ci-e2e-test resourceVersion: "27993" uid: 2f1554f5-6b51-42af-884d-ef05b74e17d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:29:06.932462Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27689" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0cb05-predictor-75df596dd-w7jqg to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:29:06Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a757bc2c9f5 namespace: kserve-ci-e2e-test resourceVersion: "27694" uid: adb2d5d7-ddc4-448b-b58b-c1b56321d78b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0cb05-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:07Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a758745aee1 namespace: kserve-ci-e2e-test resourceVersion: "27713" uid: 0c2dcb5f-923c-4129-a29e-3ef5d3f011f8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27696" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75c1bb31c0 namespace: kserve-ci-e2e-test resourceVersion: "27746" uid: fe00f671-5cf7-412b-8efa-43d007b4a1ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75c337d2dd namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: 4f25ca03-5987-4469-a5e0-5699323e2a93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75c68c2e87 namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: 7a6ae704-df67-4d3c-bf60-e151906d910c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75c7136637 namespace: kserve-ci-e2e-test resourceVersion: "27752" uid: fd8e752f-fd82-4698-8c6b-7f3a08399160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75c71ecdab namespace: kserve-ci-e2e-test resourceVersion: "27753" uid: 4c242b80-1350-486f-83f0-1050cf24fdcf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75cad8388c namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: e7e3da3e-d4a8-4aeb-b2ef-daad0f8bb421 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a75cb4c44a8 namespace: kserve-ci-e2e-test resourceVersion: "27757" uid: 91d7e409-7dde-4504-9a34-8765f4a03ae7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:29:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:55Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67a761f23e845 namespace: kserve-ci-e2e-test resourceVersion: "28227" uid: 9aa8d8b6-f390-4b5b-a5ef-5865c0ea1cc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:46:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:37Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67b6a00dfe1b8 namespace: kserve-ci-e2e-test resourceVersion: "35473" uid: 039ec6e8-fd43-4735-bce7-5e6a4866c626 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:46:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:37Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67b6a00e05a41 namespace: kserve-ci-e2e-test resourceVersion: "35474" uid: 012b8037-c4fd-4970-a537-b8966247b3e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: fdf0fc94-7aca-4994-a00c-ac28739a3c80 kind: Event lastTimestamp: "2026-09-18T17:46:40Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:46:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:40Z" name: success-200-isvc-0cb05-predictor-75df596dd-w7jqg.18d67b6ad3dc9f7c namespace: kserve-ci-e2e-test resourceVersion: "35550" uid: 4370e1a5-3b7e-40ea-bc85-933cd58d8153 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0cb05-predictor-75df596dd namespace: kserve-ci-e2e-test resourceVersion: "27683" uid: 940ce39b-9591-4398-b73a-b44fe3031b28 kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: 'Created pod: success-200-isvc-0cb05-predictor-75df596dd-w7jqg' metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:06Z" name: success-200-isvc-0cb05-predictor-75df596dd.18d67a757b44cb02 namespace: kserve-ci-e2e-test resourceVersion: "27690" uid: 2c1a34cb-7520-4723-be9e-314f2ad66fb8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27681" uid: 8b931392-6392-4548-998a-c0b07239be21 kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: Scaled up replica set success-200-isvc-0cb05-predictor-75df596dd from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:06Z" name: success-200-isvc-0cb05-predictor.18d67a7578888120 namespace: kserve-ci-e2e-test resourceVersion: "27685" uid: bbad2c37-f1d0-4673-ab97-df5c6d7ac78c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:29:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: 6af8cc36-f268-4a85-98b5-e643bf625486 kind: Event lastTimestamp: "2026-09-18T17:29:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:51Z" name: success-200-isvc-0cb05-predictor.18d67a78f6c1c805 namespace: kserve-ci-e2e-test resourceVersion: "28184" uid: 02a4c84e-68ef-47c3-b2ef-f95b3678297e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:29:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: 6af8cc36-f268-4a85-98b5-e643bf625486 kind: Event lastTimestamp: "2026-09-18T17:29:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:51Z" name: success-200-isvc-0cb05-predictor.18d67a78f7136684 namespace: kserve-ci-e2e-test resourceVersion: "28185" uid: 10c1793e-6307-47a5-97c0-bebb1ffb5900 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27890" uid: 6af8cc36-f268-4a85-98b5-e643bf625486 kind: Event lastTimestamp: "2026-09-18T17:30:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:21Z" name: success-200-isvc-0cb05-predictor.18d67a7c75d73cd8 namespace: kserve-ci-e2e-test resourceVersion: "28628" uid: 86336607-de33-49b7-8209-8accad76df18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:29:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0cb05-predictor namespace: kserve-ci-e2e-test resourceVersion: "27890" uid: 6af8cc36-f268-4a85-98b5-e643bf625486 kind: Event lastTimestamp: "2026-09-18T17:30:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:21Z" name: success-200-isvc-0cb05-predictor.18d67a7c764e4831 namespace: kserve-ci-e2e-test resourceVersion: "28629" uid: 94725374-8e6f-473f-b2d7-7ac7237d330c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "27669" uid: 4e9550f9-28d0-44e6-9657-670a4e1be0a4 kind: Event lastTimestamp: "2026-09-18T17:30:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:05Z" name: success-200-isvc-0cb05.18d67a757869b10a namespace: kserve-ci-e2e-test resourceVersion: "28467" uid: 94f28a5f-1b94-4b25-9b96-6d0cf0741567 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "28464" uid: 4e9550f9-28d0-44e6-9657-670a4e1be0a4 kind: Event lastTimestamp: "2026-09-18T17:30:05Z" message: InferenceService [success-200-isvc-0cb05] is Ready metadata: creationTimestamp: "2026-09-18T17:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:05Z" name: success-200-isvc-0cb05.18d67a83336891d8 namespace: kserve-ci-e2e-test resourceVersion: "28466" uid: e6a5fd4f-c691-42ee-90df-34ae791d7bdc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:08:23.975547Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1ff44-predictor-54b47c469d-hwr25 to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:08:23Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d6795415c155cf namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 950c32a0-a458-47a4-a48e-8444f1803b67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1ff44-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:24Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d6795423361c58 namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 3101cb23-57bf-44fe-8227-b6f77702e650 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d679545c2e6071 namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: e8076a19-e51b-45c9-923b-bd40ed24fa09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d679545e523599 namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: ebeb5ca7-67ce-412e-85d2-ad2efbd4da46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d6795461484120 namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: 14d29d5d-0716-434d-966f-01d118071304 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d6795461c38fc8 namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: b1d4c738-1306-4d55-9d0a-6b1b16922ecb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d6795461cea25c namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: a0efdb0f-389f-455c-b3a6-048ca50124bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d6795465c6294c namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: 8043521b-cc13-4cc1-ac6e-87a3c09e53cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:08:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:25Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d67954663cb128 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: fd260979-0b59-4594-94a9-d2b73a181ae7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:17:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:52Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d67954afbce636 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 48174f48-4ab5-4868-9fc2-6bb571342f65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:49Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d679d7b2dcadfd namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: a15e5afd-dc3a-4faf-af9e-bf8c1c55d861 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:49Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d679d7b2dd2371 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: 423b5dc2-1567-47b3-8564-191242920cdf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cf1783d6-7404-4a56-baad-1858de663182 kind: Event lastTimestamp: "2026-09-18T17:17:52Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:52Z" name: success-200-isvc-1ff44-predictor-54b47c469d-hwr25.18d679d8785cc689 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 4dfd72fa-8986-436f-8cbd-b3659580b923 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1ff44-predictor-54b47c469d namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: 79e0375d-2c61-4bae-a6f5-4d3f17702d2f kind: Event lastTimestamp: "2026-09-18T17:08:23Z" message: 'Created pod: success-200-isvc-1ff44-predictor-54b47c469d-hwr25' metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:23Z" name: success-200-isvc-1ff44-predictor-54b47c469d.18d6795415991f81 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: b1088423-55ab-4623-87df-5c7d86110cbb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17107" uid: 9cbf7a76-be1f-459e-ba01-bf63744815a5 kind: Event lastTimestamp: "2026-09-18T17:08:23Z" message: Scaled up replica set success-200-isvc-1ff44-predictor-54b47c469d from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:23Z" name: success-200-isvc-1ff44-predictor.18d679540ec2cbfb namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: a1f4866c-2e09-429e-a8e4-f70fb63873be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:08:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17108" uid: d0c46d3d-5826-49bf-9f54-a96c3a2c4bfb kind: Event lastTimestamp: "2026-09-18T17:08:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:53Z" name: success-200-isvc-1ff44-predictor.18d679578e0a4e5c namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: a5c3f036-cead-4a79-b405-419d4f741eb7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:08:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17108" uid: d0c46d3d-5826-49bf-9f54-a96c3a2c4bfb kind: Event lastTimestamp: "2026-09-18T17:08:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:53Z" name: success-200-isvc-1ff44-predictor.18d679578e7e58b1 namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: b9f2fdc0-bbdc-4697-8998-e451a61a7087 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:09:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: d0c46d3d-5826-49bf-9f54-a96c3a2c4bfb kind: Event lastTimestamp: "2026-09-18T17:09:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:39Z" name: success-200-isvc-1ff44-predictor.18d6795e8dd6423b namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: 511dee0a-3d01-4ea6-8018-3c7b5d023517 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:09:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ff44-predictor namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: d0c46d3d-5826-49bf-9f54-a96c3a2c4bfb kind: Event lastTimestamp: "2026-09-18T17:09:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:39Z" name: success-200-isvc-1ff44-predictor.18d6795e8e87cf65 namespace: kserve-ci-e2e-test resourceVersion: "18093" uid: 0f6f6481-aa54-42c9-94ab-17b451287e1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:08:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 14f9c584-b6ff-4db9-8d1a-c403f8b28730 kind: Event lastTimestamp: "2026-09-18T17:09:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:22Z" name: success-200-isvc-1ff44.18d679540ed6360c namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 3f04468f-3510-4190-b7fa-ac96f42c61d5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "17920" uid: 14f9c584-b6ff-4db9-8d1a-c403f8b28730 kind: Event lastTimestamp: "2026-09-18T17:09:22Z" message: InferenceService [success-200-isvc-1ff44] is Ready metadata: creationTimestamp: "2026-09-18T17:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:22Z" name: success-200-isvc-1ff44.18d67961c113b0bf namespace: kserve-ci-e2e-test resourceVersion: "17922" uid: 6db425bd-76a7-4217-9f55-5813332b081d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:17:49.428929Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-21436-predictor-c549db5dd-4klsl to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:17:49Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7bd6738ee namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 3d2e7f57-9e0f-41b1-938a-a4a51d6d2982 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7df8ba3bf namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: b37e185d-daf8-40b4-8242-a4a27fca3466 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7e160733a namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 80bfc471-dccd-4c98-b59d-372ff5333571 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7e43b52d0 namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: 707325f1-a6dc-4e41-9b22-42d992b3bd18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7e4b43d45 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: 4e5e4f4b-82a5-49cc-b9a8-67ca03092307 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7e4c02412 namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: cdaa7cbe-8b96-41e5-a4b9-eb4ab8d9e056 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7e7f67d8e namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 11922687-66e9-4f10-bc2a-2e068c6ee20a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:17:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:50Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7e875664d namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: ef3d32f7-41cd-4df5-b7ef-3b6d6f0062a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:18:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:36Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679d7f259a75e namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 05f25a04-0cb0-4b95-af63-65b857c4fbf0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:10Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679ea92a82c3d namespace: kserve-ci-e2e-test resourceVersion: "22547" uid: e7edb4db-465d-4362-8979-7bc8a5a10458 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:10Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679ea92a8b7ca namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: e8013435-bd84-4cce-a0ba-6f17d09bfdc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-21436-predictor-c549db5dd-4klsl namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: b17786bb-e59e-4104-ae4c-65e0e3db8256 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-21436-predictor-c549db5dd-4klsl.18d679eacec3ea18 namespace: kserve-ci-e2e-test resourceVersion: "22628" uid: ed4b65d2-fd05-4e19-b8c5-c1e1b4fc711a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-21436-predictor-c549db5dd namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 0208436b-ab5b-4139-bda9-736d92a536ba kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: 'Created pod: success-200-isvc-21436-predictor-c549db5dd-4klsl' metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:17:49Z" name: success-200-isvc-21436-predictor-c549db5dd.18d679d7bc97afa2 namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 464c06c7-82b5-4278-a70f-cec62da49e23 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 308ab630-dc71-4699-895d-d204b4e03bd5 kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Scaled up replica set success-200-isvc-21436-predictor-c549db5dd from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:17:49Z" name: success-200-isvc-21436-predictor.18d679d7b7c56699 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 5b89de01-f986-456b-90ad-07fac65ed27e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:18:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: 41937bc7-613f-4b74-b8e5-83d3bb48c804 kind: Event lastTimestamp: "2026-09-18T17:18:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:34Z" name: success-200-isvc-21436-predictor.18d679db364d4e51 namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: daec8d24-4244-4b24-9e6a-5ccd506569e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:18:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: 41937bc7-613f-4b74-b8e5-83d3bb48c804 kind: Event lastTimestamp: "2026-09-18T17:18:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:34Z" name: success-200-isvc-21436-predictor.18d679db36b42adf namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: 01605708-3313-4d66-b982-ac4707adbe12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:18:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 41937bc7-613f-4b74-b8e5-83d3bb48c804 kind: Event lastTimestamp: "2026-09-18T17:19:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:04Z" name: success-200-isvc-21436-predictor.18d679deb5afb342 namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 7fab133b-da07-4407-8dc3-fa1c6d9c8a76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:18:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-21436-predictor namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 41937bc7-613f-4b74-b8e5-83d3bb48c804 kind: Event lastTimestamp: "2026-09-18T17:19:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:04Z" name: success-200-isvc-21436-predictor.18d679deb61da7e3 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: 075c52b7-97be-4262-9d0b-4e09b34a9dba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21436 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: a1479551-ff2f-4eed-b78e-ee1f3e69cc7d kind: Event lastTimestamp: "2026-09-18T17:18:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:46Z" name: success-200-isvc-21436.18d679d7b7e2689b namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 3daa93cb-6755-46e7-b45e-f59c4006c082 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-21436 namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: a1479551-ff2f-4eed-b78e-ee1f3e69cc7d kind: Event lastTimestamp: "2026-09-18T17:18:46Z" message: InferenceService [success-200-isvc-21436] is Ready metadata: creationTimestamp: "2026-09-18T17:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:18:46Z" name: success-200-isvc-21436.18d679e5027bb394 namespace: kserve-ci-e2e-test resourceVersion: "22261" uid: fb77d852-b2c1-42d1-9444-4858e8af5491 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:18:20.146149Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78e6b-predictor-6cd8944648-5vd78 to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:18:20Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679dee44ac27d namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 01fd6ec5-a9d0-4016-978d-e6c28eeb5350 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-78e6b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679deec56cdb9 namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: c9e36387-cfc4-42c6-8085-aeb7a6be722b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df2a9a2bc5 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: 93aa97aa-1ea4-45a7-91b1-5fe0c1fe5345 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df2c2e1dac namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 59ba50ff-5418-4128-ba68-29554d31169b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df2f500d9c namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: 0548fcb1-21c6-4572-8873-a41f7b79afd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df2fc3c283 namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: 9fbc8d3e-4d30-4eff-be8f-cc9b136d490f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df2fcec84d namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 72ed6790-a7b1-46bd-9baa-bde18ec3fe00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df3478a8a7 namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: e4bef408-686c-403d-a593-ca2e73f8a6c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679df34f886a8 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 63ac95f9-9b35-48c6-9e8d-852477de48e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:19:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:09Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679dfac8657fa namespace: kserve-ci-e2e-test resourceVersion: "22500" uid: a4c67117-a57c-4307-981a-68f48aee1251 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679f1c64613e7 namespace: kserve-ci-e2e-test resourceVersion: "22993" uid: 5955f5ac-5528-4207-b36f-df3f4674575a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679f1c64789a4 namespace: kserve-ci-e2e-test resourceVersion: "23001" uid: 104d6ab2-e57c-4c84-8dfb-a4b71b7202ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 904d024d-55c0-4f00-89ad-1a11e61dfddb kind: Event lastTimestamp: "2026-09-18T17:19:44Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:44Z" name: success-200-isvc-78e6b-predictor-6cd8944648-5vd78.18d679f288de6f43 namespace: kserve-ci-e2e-test resourceVersion: "23122" uid: bb9d9091-56ab-4412-b104-d96899e86488 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78e6b-predictor-6cd8944648 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: b82a8d26-b007-46c6-a692-e26abcf50f76 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: 'Created pod: success-200-isvc-78e6b-predictor-6cd8944648-5vd78' metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:20Z" name: success-200-isvc-78e6b-predictor-6cd8944648.18d679dee3d6eeaf namespace: kserve-ci-e2e-test resourceVersion: "21966" uid: 551fec24-60df-435c-b8f9-7fd98b245196 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 86dacc68-e93b-4340-9068-96e03f066c16 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Scaled up replica set success-200-isvc-78e6b-predictor-6cd8944648 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:18:20Z" name: success-200-isvc-78e6b-predictor.18d679dee1fc3833 namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: 01bc7cd8-dfe9-4715-a13e-0bcb72a642ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:18:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: 6c4a80f4-1422-4697-83cc-ca11eb0c27e8 kind: Event lastTimestamp: "2026-09-18T17:19:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:05Z" name: success-200-isvc-78e6b-predictor.18d679e2607a6c83 namespace: kserve-ci-e2e-test resourceVersion: "22458" uid: 8cfa2c8f-08fa-4fcf-a301-74619fe2519c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:18:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: 6c4a80f4-1422-4697-83cc-ca11eb0c27e8 kind: Event lastTimestamp: "2026-09-18T17:19:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:18:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:05Z" name: success-200-isvc-78e6b-predictor.18d679e260df6df9 namespace: kserve-ci-e2e-test resourceVersion: "22459" uid: 65ae3707-22d7-440a-8763-cb9d9cbaa34b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 6c4a80f4-1422-4697-83cc-ca11eb0c27e8 kind: Event lastTimestamp: "2026-09-18T17:19:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:35Z" name: success-200-isvc-78e6b-predictor.18d679ecdd66a207 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: de4da3f2-001b-4215-bf36-d5b36b13df37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78e6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 6c4a80f4-1422-4697-83cc-ca11eb0c27e8 kind: Event lastTimestamp: "2026-09-18T17:19:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:35Z" name: success-200-isvc-78e6b-predictor.18d679ecde22f656 namespace: kserve-ci-e2e-test resourceVersion: "22897" uid: 9106e0c9-2636-46c9-8118-fadfc51f253a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78e6b namespace: kserve-ci-e2e-test resourceVersion: "21943" uid: 250d82fc-b620-4a35-aaff-5006bae999cb kind: Event lastTimestamp: "2026-09-18T17:19:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:19Z" name: success-200-isvc-78e6b.18d679dee210ba31 namespace: kserve-ci-e2e-test resourceVersion: "22742" uid: 4ab79f8b-95e8-454a-8102-239a0775b005 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78e6b namespace: kserve-ci-e2e-test resourceVersion: "22739" uid: 250d82fc-b620-4a35-aaff-5006bae999cb kind: Event lastTimestamp: "2026-09-18T17:19:19Z" message: InferenceService [success-200-isvc-78e6b] is Ready metadata: creationTimestamp: "2026-09-18T17:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:19Z" name: success-200-isvc-78e6b.18d679ecbe03118a namespace: kserve-ci-e2e-test resourceVersion: "22740" uid: 300f9d14-4ad8-4117-9ff3-7f8710ead6b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:19:41.457990Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23032" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1d2daf5d9 namespace: kserve-ci-e2e-test resourceVersion: "23036" uid: 503e172f-ea0a-4e8b-aa6b-6271ea9011f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23042" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1f5056027 namespace: kserve-ci-e2e-test resourceVersion: "23079" uid: 1c99be90-7855-4d96-ba09-15ad43777152 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1f6cbf3ba namespace: kserve-ci-e2e-test resourceVersion: "23081" uid: 8da4319d-fcc0-499f-8707-33dacd50c199 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1f9e86d80 namespace: kserve-ci-e2e-test resourceVersion: "23083" uid: 7206099a-0432-411d-9a8f-b8ed3395eb71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1fa58fc75 namespace: kserve-ci-e2e-test resourceVersion: "23084" uid: 136f3f16-aa56-425d-934d-2fcdc3781c10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1fa636ce9 namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: 19205821-feb9-4ba9-8721-0d15229c087f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1fda734de namespace: kserve-ci-e2e-test resourceVersion: "23087" uid: 0f5bb5c2-e373-4497-9942-7e3631f79f6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:19:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:42Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f1fe1a4343 namespace: kserve-ci-e2e-test resourceVersion: "23088" uid: bf349dbe-4b55-497d-a70d-94436c08fdaa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d679f229b1c14c namespace: kserve-ci-e2e-test resourceVersion: "27771" uid: 1f1d9819-4f02-4766-9201-37900731e397 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:06Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d67a75711fa8f5 namespace: kserve-ci-e2e-test resourceVersion: "27655" uid: e2c87624-ae20-4992-9173-7d108c24d8b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:29:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:06Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d67a7571204576 namespace: kserve-ci-e2e-test resourceVersion: "27661" uid: d3837d8e-ceb1-410c-918d-8e34741383a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: c9bf7cd4-8630-4584-bee8-5de542d492bb kind: Event lastTimestamp: "2026-09-18T17:29:08Z" message: 'Readiness probe failed: Get "https://10.132.0.44:8643/healthz": dial tcp 10.132.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:08Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv.18d67a75f2352300 namespace: kserve-ci-e2e-test resourceVersion: "27770" uid: aba1c46e-bf4c-4053-b3f3-2fe88f691095 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-792b5-predictor-8d4d6d7cd namespace: kserve-ci-e2e-test resourceVersion: "23025" uid: 97eeecb2-52c5-4c76-ae84-f9af7f5a89de kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: 'Created pod: success-200-isvc-792b5-predictor-8d4d6d7cd-spnxv' metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-792b5-predictor-8d4d6d7cd.18d679f1d25fbfd1 namespace: kserve-ci-e2e-test resourceVersion: "23033" uid: 95345099-7281-49aa-896e-faacf67bcb80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23023" uid: 654f3176-7396-4062-ac33-00d63d7697d6 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: Scaled up replica set success-200-isvc-792b5-predictor-8d4d6d7cd from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-792b5-predictor.18d679f1cf60eed5 namespace: kserve-ci-e2e-test resourceVersion: "23028" uid: 8ba8f961-c2e3-4a1c-b594-8e61fbf09b6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23024" uid: ea9dca8f-46f7-4a30-80f2-0ac8ff8c74d4 kind: Event lastTimestamp: "2026-09-18T17:20:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:11Z" name: success-200-isvc-792b5-predictor.18d679f54e0a9c64 namespace: kserve-ci-e2e-test resourceVersion: "23369" uid: f0072f74-35d3-439b-a153-f4306270fe10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23024" uid: ea9dca8f-46f7-4a30-80f2-0ac8ff8c74d4 kind: Event lastTimestamp: "2026-09-18T17:20:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:11Z" name: success-200-isvc-792b5-predictor.18d679f54e7a1384 namespace: kserve-ci-e2e-test resourceVersion: "23370" uid: 98c8707c-af21-4300-b478-91cf10b7d805 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:20:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23225" uid: ea9dca8f-46f7-4a30-80f2-0ac8ff8c74d4 kind: Event lastTimestamp: "2026-09-18T17:20:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:56Z" name: success-200-isvc-792b5-predictor.18d679fc4cad9622 namespace: kserve-ci-e2e-test resourceVersion: "23801" uid: bf0707b3-ce46-4970-9683-9ced2a3340ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:20:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-792b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "23225" uid: ea9dca8f-46f7-4a30-80f2-0ac8ff8c74d4 kind: Event lastTimestamp: "2026-09-18T17:20:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:56Z" name: success-200-isvc-792b5-predictor.18d679fc4d422207 namespace: kserve-ci-e2e-test resourceVersion: "23802" uid: ae886bad-268f-42f3-bb40-6628759c571e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23013" uid: 61a21b1a-6627-4c35-8cc8-bbdfadbfc3e4 kind: Event lastTimestamp: "2026-09-18T17:20:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:39Z" name: success-200-isvc-792b5.18d679f1cf6de30f namespace: kserve-ci-e2e-test resourceVersion: "23630" uid: f2b1daaa-ac11-4d01-a2ec-6d361295e1bb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23013" uid: 61a21b1a-6627-4c35-8cc8-bbdfadbfc3e4 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-792b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-792b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-792b5.18d679f1d01f347c namespace: kserve-ci-e2e-test resourceVersion: "23029" uid: 1b37d366-a38d-4fd5-94a8-44126889457e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23013" uid: 61a21b1a-6627-4c35-8cc8-bbdfadbfc3e4 kind: Event lastTimestamp: "2026-09-18T17:19:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-792b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T17:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:19:41Z" name: success-200-isvc-792b5.18d679f1d01f9851 namespace: kserve-ci-e2e-test resourceVersion: "23031" uid: 8d5fc904-825a-4cce-a2e2-9df8aa55324b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-792b5 namespace: kserve-ci-e2e-test resourceVersion: "23627" uid: 61a21b1a-6627-4c35-8cc8-bbdfadbfc3e4 kind: Event lastTimestamp: "2026-09-18T17:20:39Z" message: InferenceService [success-200-isvc-792b5] is Ready metadata: creationTimestamp: "2026-09-18T17:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:39Z" name: success-200-isvc-792b5.18d679ff3a3b236a namespace: kserve-ci-e2e-test resourceVersion: "23628" uid: 66ebbae7-682e-4c6a-a48f-63f719f67a26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:08:54.528657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17612" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9846a-predictor-84848664-z4zxv to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:08:54Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b32dcc657 namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: c8bb3c25-7cd1-4839-8956-1792fcae7961 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17622" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b55c7d45a namespace: kserve-ci-e2e-test resourceVersion: "17660" uid: ba4054ab-2a2b-4ddf-aa80-6610cd06b74b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b578b1a9d namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: b0b5cd80-99fa-45a3-8a84-a04d8fef33ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b5aa3cfca namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: 6743d2f4-a6f1-449d-beca-26b1fcedb493 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b5b186a3f namespace: kserve-ci-e2e-test resourceVersion: "17666" uid: bc2d2553-b901-4308-868a-adb05753da7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b5b22a6d5 namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: 9895e407-899d-46ea-8b0f-a1e8e73b95c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b5de9cc51 namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: 08e35e6b-6491-41f6-a4ee-369fdd8f8014 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:08:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:55Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b5e5fe6eb namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: 7dc7b0d1-1a5e-4f23-aeae-a0b5e2d0ffbb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:08:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d6795b79b442b5 namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 3a53c42e-2587-4d35-b426-ad6cf9b3c0c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d679dedd320cc5 namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: 94323469-9ddd-44d3-abe9-73c284ba9e75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:18:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:20Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d679dedd329cd4 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 649d3c5f-ec56-48eb-b63d-c2e00d861023 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9846a-predictor-84848664-z4zxv namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: f2712b96-4f47-4fb0-9e76-3c9458bf7bc8 kind: Event lastTimestamp: "2026-09-18T17:18:21Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:21Z" name: success-200-isvc-9846a-predictor-84848664-z4zxv.18d679df42246c4a namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: 960e4186-bf5c-4d06-bbbf-d21fdce2d4bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9846a-predictor-84848664 namespace: kserve-ci-e2e-test resourceVersion: "17606" uid: bd89800e-3a78-4bb2-9bfd-6231e3e24286 kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: 'Created pod: success-200-isvc-9846a-predictor-84848664-z4zxv' metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:54Z" name: success-200-isvc-9846a-predictor-84848664.18d6795b325f4008 namespace: kserve-ci-e2e-test resourceVersion: "17615" uid: bf0b26a0-b0e9-4d6e-a868-792183183ab5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17604" uid: 2f0078c9-3270-4dec-9f57-99026bcddf4d kind: Event lastTimestamp: "2026-09-18T17:08:54Z" message: Scaled up replica set success-200-isvc-9846a-predictor-84848664 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:54Z" name: success-200-isvc-9846a-predictor.18d6795b2fc6149d namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: 69fe6d7a-35e8-460a-bf76-6c22e6a2183b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: 7a3c3d37-b6fe-4fc7-8f41-69dd977f48e7 kind: Event lastTimestamp: "2026-09-18T17:10:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:09Z" name: success-200-isvc-9846a-predictor.18d6795eae8e1b84 namespace: kserve-ci-e2e-test resourceVersion: "18387" uid: e6fd9d1b-d336-4a73-88a7-58017897d7ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: 7a3c3d37-b6fe-4fc7-8f41-69dd977f48e7 kind: Event lastTimestamp: "2026-09-18T17:10:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:09Z" name: success-200-isvc-9846a-predictor.18d6795eaf006a85 namespace: kserve-ci-e2e-test resourceVersion: "18388" uid: 175632c4-1ab7-4f4b-8804-d79b3cf0cdff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17811" uid: 7a3c3d37-b6fe-4fc7-8f41-69dd977f48e7 kind: Event lastTimestamp: "2026-09-18T17:09:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:24Z" name: success-200-isvc-9846a-predictor.18d679622de07c42 namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: 4774266f-43dd-482c-ab8e-76a93221aab2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9846a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17811" uid: 7a3c3d37-b6fe-4fc7-8f41-69dd977f48e7 kind: Event lastTimestamp: "2026-09-18T17:09:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:24Z" name: success-200-isvc-9846a-predictor.18d679622e523f86 namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: 22eebf2b-2d1b-442b-8b7a-436bcc636210 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:08:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9846a namespace: kserve-ci-e2e-test resourceVersion: "17590" uid: dd63c6bb-0a25-44e9-8b18-951717c76fad kind: Event lastTimestamp: "2026-09-18T17:09:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:51Z" name: success-200-isvc-9846a.18d6795b2fc8ee6c namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: 5ce0e622-e051-49b6-b2f8-fcddcd8cabcc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9846a namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: dd63c6bb-0a25-44e9-8b18-951717c76fad kind: Event lastTimestamp: "2026-09-18T17:09:51Z" message: InferenceService [success-200-isvc-9846a] is Ready metadata: creationTimestamp: "2026-09-18T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:51Z" name: success-200-isvc-9846a.18d679688bac6811 namespace: kserve-ci-e2e-test resourceVersion: "18204" uid: f9d83838-6191-4e90-a512-4abe3e8438ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:06:32.820646Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a4cbd-predictor-68d958b656-ld56h to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:06:32Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793a346829a9 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 5869a34d-854e-4a45-a398-524a7813c1f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a4cbd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:33Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793a3cb999e9 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: d990d66a-3762-4a77-a7ac-65cf646ac9d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:06:34Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793ab08b0160 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 03287a94-51fd-471b-93ad-4b637e9bd064 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:34Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" metadata: creationTimestamp: "2026-09-18T17:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:34Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793ab224689e namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: b8b8851e-7a3d-4338-bed6-0e9d9ce5a82d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" in 18.739s (18.739s including waiting). Image size: 1384255053 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793f0f15c079 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 10b94748-18c6-4186-9b06-4295c16d1dfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793f12e7665e namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 1f8048e9-aaf0-4f44-b0a7-9b10e3f8e375 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793f13539799 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 0e210ea2-8eb6-4d9f-8201-6768d5a5e805 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:53Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T17:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:53Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793f13612c36 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: eead0441-cb21-4ac8-a96d-f474586e435f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.535s (2.535s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T17:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:56Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793faa80824d namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 44b10f82-fc4b-4894-8998-6ad859f2ae0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:56Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793fae2c5be6 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 5935df77-0d8c-4a66-9131-d6deb328ef6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:06:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:56Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793fae99c11a namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 2d7a3f0f-81bf-492a-9a6b-b33214ddee15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T17:06:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:07:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:06:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:43Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6793fdc27be60 namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: f88c26f0-8a2f-44b4-9d2e-1d85ed321938 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:08:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:23Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6795411947eab namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: 7f542c92-a607-4abd-8803-77bfd98e25ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 118cb984-b1e6-4cc5-aa53-755b241ff683 kind: Event lastTimestamp: "2026-09-18T17:08:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:23Z" name: success-200-isvc-a4cbd-predictor-68d958b656-ld56h.18d6795411957200 namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: 653b6edb-dd1d-436c-b364-86ba64882d76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a4cbd-predictor-68d958b656 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 4aab3b98-bee3-467d-b1f2-a6a3ccfd5706 kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: 'Created pod: success-200-isvc-a4cbd-predictor-68d958b656-ld56h' metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:32Z" name: success-200-isvc-a4cbd-predictor-68d958b656.18d6793a32a7e613 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 41f7890b-9169-426c-b425-2284ecd87eb6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: c859c25d-ab22-4e7d-9b86-dd2a4d89e3ab kind: Event lastTimestamp: "2026-09-18T17:06:32Z" message: Scaled up replica set success-200-isvc-a4cbd-predictor-68d958b656 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:06:32Z" name: success-200-isvc-a4cbd-predictor.18d6793a304bc029 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 72b3a938-97ef-44a3-a6f2-b051b84c8dfb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: b85e6d63-e2af-430e-a988-6fc8abce17f8 kind: Event lastTimestamp: "2026-09-18T17:07:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:32Z" name: success-200-isvc-a4cbd-predictor.18d6793daec7128a namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: 02281da8-03c0-4b1f-885c-0c541bfc77f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T17:06:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: b85e6d63-e2af-430e-a988-6fc8abce17f8 kind: Event lastTimestamp: "2026-09-18T17:07:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:07:32Z" name: success-200-isvc-a4cbd-predictor.18d6793daf206122 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: 62f6521d-2473-49c0-8826-4b4bf73a5711 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:07:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: b85e6d63-e2af-430e-a988-6fc8abce17f8 kind: Event lastTimestamp: "2026-09-18T17:08:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:17Z" name: success-200-isvc-a4cbd-predictor.18d6794baa415d76 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 69629e72-c6f7-4106-8e69-f84a91d8a01e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:07:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a4cbd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: b85e6d63-e2af-430e-a988-6fc8abce17f8 kind: Event lastTimestamp: "2026-09-18T17:08:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:17Z" name: success-200-isvc-a4cbd-predictor.18d6794baaae4677 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: bc02c334-281b-4515-991b-ba21c1aa72e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:06:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a4cbd namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: af9e2286-078b-4e3c-b921-f9713d71f993 kind: Event lastTimestamp: "2026-09-18T17:07:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:07:53Z" name: success-200-isvc-a4cbd.18d6793a30563499 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: b9fbb7ba-7a41-4620-8cf7-3f75c497553b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:07:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a4cbd namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: af9e2286-078b-4e3c-b921-f9713d71f993 kind: Event lastTimestamp: "2026-09-18T17:07:53Z" message: InferenceService [success-200-isvc-a4cbd] is Ready metadata: creationTimestamp: "2026-09-18T17:07:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:07:53Z" name: success-200-isvc-a4cbd.18d6794cebbceb14 namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: f95be04d-d5a5-46bc-950b-4202e5e85a20 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:19:10.437376Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22563" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:19:10Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679ea99e20f5f namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: a55cac98-7604-42b8-b09d-ce09919a40b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-cca0b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:10Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eaa5243938 namespace: kserve-ci-e2e-test resourceVersion: "22603" uid: 0c24c839-c768-4128-811d-a1a28b60e064 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22579" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eadfc60157 namespace: kserve-ci-e2e-test resourceVersion: "22638" uid: 294ebaf8-44c2-4e1b-941d-da941fdd33e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eae1be9891 namespace: kserve-ci-e2e-test resourceVersion: "22640" uid: 4bf0e6fd-2666-4bca-8e45-da69ee849a52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eae4bdbf9c namespace: kserve-ci-e2e-test resourceVersion: "22641" uid: 7fa5fac3-c64f-46a6-ac57-44c101e084b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eae5326fa1 namespace: kserve-ci-e2e-test resourceVersion: "22642" uid: 2975f673-38d4-4241-ab51-999feb3adb25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eae53d4036 namespace: kserve-ci-e2e-test resourceVersion: "22643" uid: 482e306c-4c20-4c86-bbd3-076dbf3abf9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eae84c192e namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: 0696c6be-76fd-4cca-98c0-e0a4b78b9280 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:19:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:19:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:19:11Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eae8c0ba47 namespace: kserve-ci-e2e-test resourceVersion: "22646" uid: 6efa770e-7359-4498-80d3-916c32b066ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:19:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:28:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:38Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d679eb2438e22f namespace: kserve-ci-e2e-test resourceVersion: "27391" uid: e53aaac5-064c-41d6-8164-8d21178a258b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:35Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d67a6e3738d48f namespace: kserve-ci-e2e-test resourceVersion: "27263" uid: a3a8ea8b-322f-4df7-b15f-7d4253b3be5b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:28:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:35Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d67a6e37397670 namespace: kserve-ci-e2e-test resourceVersion: "27269" uid: 4a96771f-4da1-4462-ad78-24012d2eda79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: f8eab596-268c-4b37-9bd0-8f2084a18df9 kind: Event lastTimestamp: "2026-09-18T17:28:38Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:28:38Z" name: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts.18d67a6eecb14c4c namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: a29253dd-f9b2-4cb3-9ed4-57b167f60b73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cca0b-predictor-6bf79666fd namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: 38f88da2-eaaf-4888-9ab6-7be0aa6345dc kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: 'Created pod: success-200-isvc-cca0b-predictor-6bf79666fd-7q7ts' metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:10Z" name: success-200-isvc-cca0b-predictor-6bf79666fd.18d679ea994f1d18 namespace: kserve-ci-e2e-test resourceVersion: "22565" uid: d020a89e-885e-4420-b89c-98f98ca4eb2b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22555" uid: ef475445-d929-435d-b2d0-3fc829f93e15 kind: Event lastTimestamp: "2026-09-18T17:19:10Z" message: Scaled up replica set success-200-isvc-cca0b-predictor-6bf79666fd from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:10Z" name: success-200-isvc-cca0b-predictor.18d679ea96bb65db namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: dd5fe24d-fc30-4233-a4cc-d582e6e9dbc4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 6a419544-a4c4-40a9-a7a4-3dd393ee9aab kind: Event lastTimestamp: "2026-09-18T17:19:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:40Z" name: success-200-isvc-cca0b-predictor.18d679ee1576d4b0 namespace: kserve-ci-e2e-test resourceVersion: "22950" uid: 181e43f1-3820-4cd4-a771-5796468d04ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:19:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 6a419544-a4c4-40a9-a7a4-3dd393ee9aab kind: Event lastTimestamp: "2026-09-18T17:19:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:19:40Z" name: success-200-isvc-cca0b-predictor.18d679ee15f76632 namespace: kserve-ci-e2e-test resourceVersion: "22951" uid: a482929e-7835-4890-a2e4-5f0d92b012bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:19:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22786" uid: 6a419544-a4c4-40a9-a7a4-3dd393ee9aab kind: Event lastTimestamp: "2026-09-18T17:20:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:25Z" name: success-200-isvc-cca0b-predictor.18d679f51483f187 namespace: kserve-ci-e2e-test resourceVersion: "23506" uid: 6da81899-f3a5-4e4a-bd57-d30ab44b6d1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:19:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cca0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22786" uid: 6a419544-a4c4-40a9-a7a4-3dd393ee9aab kind: Event lastTimestamp: "2026-09-18T17:20:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:20:25Z" name: success-200-isvc-cca0b-predictor.18d679f514ede547 namespace: kserve-ci-e2e-test resourceVersion: "23507" uid: f9aca886-4687-445f-8836-93a849e99f4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:19:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cca0b namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: ff76ce4e-372e-456d-b63a-5179eb822033 kind: Event lastTimestamp: "2026-09-18T17:20:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:19:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:08Z" name: success-200-isvc-cca0b.18d679ea97028ef6 namespace: kserve-ci-e2e-test resourceVersion: "23345" uid: bf944c70-5857-42ce-a45d-653e9e146d1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:20:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cca0b namespace: kserve-ci-e2e-test resourceVersion: "23341" uid: ff76ce4e-372e-456d-b63a-5179eb822033 kind: Event lastTimestamp: "2026-09-18T17:20:08Z" message: InferenceService [success-200-isvc-cca0b] is Ready metadata: creationTimestamp: "2026-09-18T17:20:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:20:08Z" name: success-200-isvc-cca0b.18d679f83576dbe4 namespace: kserve-ci-e2e-test resourceVersion: "23342" uid: e6cf68a6-92e1-44dc-84ba-ba0a58824286 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:29:57.031272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28304" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a8125e1dd2d namespace: kserve-ci-e2e-test resourceVersion: "28311" uid: 3a22dd27-701c-417c-bbb7-a742d4c5f299 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28314" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a8147ac917d namespace: kserve-ci-e2e-test resourceVersion: "28350" uid: d9c3a117-af61-4fa2-ab39-c9bd4205651c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1978" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a8149171338 namespace: kserve-ci-e2e-test resourceVersion: "28352" uid: b2c87982-b9bd-43cc-9c11-d700466a20b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a814cc60e62 namespace: kserve-ci-e2e-test resourceVersion: "28353" uid: 18df7195-2b4b-4243-8695-b3161a41c26f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a814d42d172 namespace: kserve-ci-e2e-test resourceVersion: "28355" uid: 84418cef-ef3f-45e2-be7e-35810078a837 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a814d4e9318 namespace: kserve-ci-e2e-test resourceVersion: "28356" uid: 50f655f1-6859-434c-a188-5c9ad60a47d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a81509bc838 namespace: kserve-ci-e2e-test resourceVersion: "28359" uid: 20f0bb45-3ad7-4ea1-8215-aeb5c6f4a91f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a815115d878 namespace: kserve-ci-e2e-test resourceVersion: "28361" uid: 1cf44130-9603-4019-ada5-810ed4c24d18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:39:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:23Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67a815b90831b namespace: kserve-ci-e2e-test resourceVersion: "32494" uid: 7fb78b5c-9835-4b96-9c32-41023b862dc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:39:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:22Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67b04b410b6b3 namespace: kserve-ci-e2e-test resourceVersion: "32461" uid: 3c60b639-9f9e-476b-b8f5-994c3cf99ef3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:39:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:22Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67b04b411ead0 namespace: kserve-ci-e2e-test resourceVersion: "32465" uid: c5dc933f-7d29-4bf6-aab4-49e2f6655e3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f namespace: kserve-ci-e2e-test resourceVersion: "28308" uid: 4379ec2e-7b0d-4ae3-bfaa-07e70ea2ca83 kind: Event lastTimestamp: "2026-09-18T17:39:23Z" message: 'Readiness probe failed: Get "https://10.132.0.47:8643/healthz": dial tcp 10.132.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:39:23Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f.18d67b05245aa26d namespace: kserve-ci-e2e-test resourceVersion: "32493" uid: fabd17f2-ef4f-47c2-9752-77169e0902e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b namespace: kserve-ci-e2e-test resourceVersion: "28298" uid: 2f3f3438-4b7b-4b04-9f06-c22115a4ae15 kind: Event lastTimestamp: "2026-09-18T17:29:57Z" message: 'Created pod: success-200-isvc-f0fb0-predictor-7cbb4c9f9b-5zb9f' metadata: creationTimestamp: "2026-09-18T17:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:57Z" name: success-200-isvc-f0fb0-predictor-7cbb4c9f9b.18d67a8124af1433 namespace: kserve-ci-e2e-test resourceVersion: "28306" uid: 1c1ffa06-af9a-48c5-8041-2892df64f82f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28295" uid: 271c9601-1dfb-40f0-bb92-990f129e82df kind: Event lastTimestamp: "2026-09-18T17:29:56Z" message: Scaled up replica set success-200-isvc-f0fb0-predictor-7cbb4c9f9b from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:29:56Z" name: success-200-isvc-f0fb0-predictor.18d67a8122ceb524 namespace: kserve-ci-e2e-test resourceVersion: "28299" uid: 91656787-6163-4fa1-a9e0-0f9464ee994c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:30:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28297" uid: 9d444934-33d0-44d5-8b32-5be518c10e1f kind: Event lastTimestamp: "2026-09-18T17:30:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:27Z" name: success-200-isvc-f0fb0-predictor.18d67a84a1526d92 namespace: kserve-ci-e2e-test resourceVersion: "28673" uid: cf4ea927-80fe-49cb-9b75-dcc65694eea5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:30:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28297" uid: 9d444934-33d0-44d5-8b32-5be518c10e1f kind: Event lastTimestamp: "2026-09-18T17:30:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:27Z" name: success-200-isvc-f0fb0-predictor.18d67a84a1b5845b namespace: kserve-ci-e2e-test resourceVersion: "28674" uid: 2cddc6c1-777d-4692-90b8-83b9078f3279 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:30:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28518" uid: 9d444934-33d0-44d5-8b32-5be518c10e1f kind: Event lastTimestamp: "2026-09-18T17:31:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:12Z" name: success-200-isvc-f0fb0-predictor.18d67a8b9f72314c namespace: kserve-ci-e2e-test resourceVersion: "29062" uid: 44f1b8e4-ac8e-4aaa-a3ff-cc1d0166f9e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T17:30:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f0fb0-predictor namespace: kserve-ci-e2e-test resourceVersion: "28518" uid: 9d444934-33d0-44d5-8b32-5be518c10e1f kind: Event lastTimestamp: "2026-09-18T17:31:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:31:12Z" name: success-200-isvc-f0fb0-predictor.18d67a8b9feda762 namespace: kserve-ci-e2e-test resourceVersion: "29063" uid: cc644b1c-3ed9-433c-9ff2-58204a8b9592 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T17:29:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28285" uid: 13f84baa-6c60-4d24-ba2e-c709015fd06a kind: Event lastTimestamp: "2026-09-18T17:30:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:54Z" name: success-200-isvc-f0fb0.18d67a8122e71484 namespace: kserve-ci-e2e-test resourceVersion: "28894" uid: 634db2c8-a1c5-4c1c-96c3-7f0ec0053bf9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f0fb0 namespace: kserve-ci-e2e-test resourceVersion: "28887" uid: 13f84baa-6c60-4d24-ba2e-c709015fd06a kind: Event lastTimestamp: "2026-09-18T17:30:54Z" message: InferenceService [success-200-isvc-f0fb0] is Ready metadata: creationTimestamp: "2026-09-18T17:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:54Z" name: success-200-isvc-f0fb0.18d67a8e71b23590 namespace: kserve-ci-e2e-test resourceVersion: "28890" uid: b1a52a53-1dc6-4d02-8ec8-f65921fa8419 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:30:16.859817Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28566" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-0cb05-676d55bd6d-klj8m to ip-10-0-142-7.ec2.internal metadata: creationTimestamp: "2026-09-18T17:30:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:30:16Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67a85c3c17fc0 namespace: kserve-ci-e2e-test resourceVersion: "28571" uid: 7c430e04-ab77-402d-8dd6-6a314ee78756 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:30:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-0cb05-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:30:17Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67a85cc22888d namespace: kserve-ci-e2e-test resourceVersion: "28582" uid: a4fb49ea-4410-4680-a056-4fe2c69cf6a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28574" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:30:18Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:30:18Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67a8609ac66b4 namespace: kserve-ci-e2e-test resourceVersion: "28590" uid: 03df7a99-125e-4fe3-a889-e75619f0d045 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0cb05} kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:30:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:30:18Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67a860af9971e namespace: kserve-ci-e2e-test resourceVersion: "28593" uid: ac243f12-f512-434e-bb34-0a8396a13ef7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0cb05} kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:30:18Z" message: 'Created container: switch-graph-0cb05' metadata: creationTimestamp: "2026-09-18T17:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:30:18Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67a860de47b0c namespace: kserve-ci-e2e-test resourceVersion: "28594" uid: 567f1640-d819-4509-9d99-b9a6b3aecba5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0cb05} kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:30:18Z" message: Started container switch-graph-0cb05 metadata: creationTimestamp: "2026-09-18T17:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:30:18Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67a860e61caa7 namespace: kserve-ci-e2e-test resourceVersion: "28595" uid: 36395f49-8e18-4dfd-99c4-b05f6ef09f84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0cb05} kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:46:36Z" message: Stopping container switch-graph-0cb05 metadata: creationTimestamp: "2026-09-18T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:46:36Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67b69f883f349 namespace: kserve-ci-e2e-test resourceVersion: "35458" uid: 20a1abdd-1618-4891-9167-2f64a55a3973 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:46:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0cb05} kind: Pod name: switch-graph-0cb05-676d55bd6d-klj8m namespace: kserve-ci-e2e-test resourceVersion: "28569" uid: 1d04580a-981a-41a6-b07e-9434bcd507eb kind: Event lastTimestamp: "2026-09-18T17:47:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:47:04Z" name: switch-graph-0cb05-676d55bd6d-klj8m.18d67b6aa07b634b namespace: kserve-ci-e2e-test resourceVersion: "35731" uid: 8c22bd80-12ac-47f7-97d8-930200c1e966 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-7.ec2.internal source: component: kubelet host: ip-10-0-142-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-0cb05-676d55bd6d namespace: kserve-ci-e2e-test resourceVersion: "28562" uid: 08ae63b9-3274-4439-a1de-f0a22e4b26dc kind: Event lastTimestamp: "2026-09-18T17:30:16Z" message: 'Created pod: switch-graph-0cb05-676d55bd6d-klj8m' metadata: creationTimestamp: "2026-09-18T17:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:16Z" name: switch-graph-0cb05-676d55bd6d.18d67a85c3419b21 namespace: kserve-ci-e2e-test resourceVersion: "28568" uid: 03d3f06e-39d9-48a1-9e3b-b56367769203 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "28560" uid: 67688b44-dbfd-4121-a8e5-b6898939a4b2 kind: Event lastTimestamp: "2026-09-18T17:30:16Z" message: Scaled up replica set switch-graph-0cb05-676d55bd6d from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:16Z" name: switch-graph-0cb05.18d67a85c2379a8a namespace: kserve-ci-e2e-test resourceVersion: "28563" uid: 7c1f1463-bb02-404c-afad-4cdcebef9795 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "28577" uid: e9b3eb33-4d4a-4839-98cb-e5cc355db751 kind: Event lastTimestamp: "2026-09-18T17:30:16Z" message: InferenceGraph [switch-graph-0cb05] is Ready metadata: creationTimestamp: "2026-09-18T17:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:30:16Z" name: switch-graph-0cb05.18d67a85c65f0f43 namespace: kserve-ci-e2e-test resourceVersion: "28579" uid: 7c3a1c00-7361-49cb-a1ab-b0f47bea871e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "28561" uid: 34c85b0e-ea78-45bb-a873-f5457623f8c3 kind: Event lastTimestamp: "2026-09-18T17:30:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:46Z" name: switch-graph-0cb05.18d67a894125eb94 namespace: kserve-ci-e2e-test resourceVersion: "28828" uid: a87f2f99-ee0d-4fb8-a1b3-07127e031b16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T17:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0cb05 namespace: kserve-ci-e2e-test resourceVersion: "28561" uid: 34c85b0e-ea78-45bb-a873-f5457623f8c3 kind: Event lastTimestamp: "2026-09-18T17:30:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:30:46Z" name: switch-graph-0cb05.18d67a8941a53559 namespace: kserve-ci-e2e-test resourceVersion: "28829" uid: fb01be70-9fa7-4c6a-af2d-19fb18e9946d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:09:33.943150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18031" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1ff44-6b78b6c5c9-7dx42 to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:09:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:09:33Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d6796460264aa8 namespace: kserve-ci-e2e-test resourceVersion: "18036" uid: 9d871b7f-9ca8-4c09-8cb8-b59809f24181 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:34Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:09:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1ff44-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T17:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:09:34Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d679646a8d2120 namespace: kserve-ci-e2e-test resourceVersion: "18049" uid: 0a333276-3f7d-4656-9be7-a3f3f1e01516 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:09:35Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:09:35Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d67964a630d8bc namespace: kserve-ci-e2e-test resourceVersion: "18058" uid: 639fb5b6-e50c-482f-9355-bf7dc6f56d63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1ff44} kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:09:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" already present on machine metadata: creationTimestamp: "2026-09-18T17:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:09:35Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d67964a86ec427 namespace: kserve-ci-e2e-test resourceVersion: "18060" uid: ee6dbad3-a701-49e8-8eee-dac0d74f162d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1ff44} kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:09:35Z" message: 'Created container: switch-graph-1ff44' metadata: creationTimestamp: "2026-09-18T17:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:09:35Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d67964abdc0753 namespace: kserve-ci-e2e-test resourceVersion: "18061" uid: 66bac0b8-bf26-4970-96a8-b248802224ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1ff44} kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:09:35Z" message: Started container switch-graph-1ff44 metadata: creationTimestamp: "2026-09-18T17:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:09:35Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d67964ac4cb0fb namespace: kserve-ci-e2e-test resourceVersion: "18062" uid: 780a68b5-31e5-4eda-ab1b-46b8979b3f64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1ff44} kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:17:49Z" message: Stopping container switch-graph-1ff44 metadata: creationTimestamp: "2026-09-18T17:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:17:49Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d679d7ab4c7fc2 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 37cd1c09-6958-4885-a68a-fb0349616203 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:17:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1ff44} kind: Pod name: switch-graph-1ff44-6b78b6c5c9-7dx42 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 470cfc94-b2b5-4781-99e8-bd1a02cd4cbc kind: Event lastTimestamp: "2026-09-18T17:18:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:17:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:18:16Z" name: switch-graph-1ff44-6b78b6c5c9-7dx42.18d679d852944550 namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: 504fe0e7-d03c-45ac-be65-5408492392d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1ff44-6b78b6c5c9 namespace: kserve-ci-e2e-test resourceVersion: "18028" uid: 43316c8a-c88c-445c-9e17-32114116fdd1 kind: Event lastTimestamp: "2026-09-18T17:09:33Z" message: 'Created pod: switch-graph-1ff44-6b78b6c5c9-7dx42' metadata: creationTimestamp: "2026-09-18T17:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:33Z" name: switch-graph-1ff44-6b78b6c5c9.18d679645fa7467a namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: b8041f3b-101f-4191-83b2-0ff2b5761436 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "18026" uid: cae93336-2f16-4506-894e-ad936a62d85d kind: Event lastTimestamp: "2026-09-18T17:09:33Z" message: Scaled up replica set switch-graph-1ff44-6b78b6c5c9 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:33Z" name: switch-graph-1ff44.18d679645dcf6b81 namespace: kserve-ci-e2e-test resourceVersion: "18029" uid: b48be3a3-56d9-438b-a23d-4973e2aa2bb8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "18043" uid: 42d4e018-463b-4049-9f8e-62a27701fb4a kind: Event lastTimestamp: "2026-09-18T17:09:34Z" message: InferenceGraph [switch-graph-1ff44] is Ready metadata: creationTimestamp: "2026-09-18T17:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:09:34Z" name: switch-graph-1ff44.18d6796466dd711a namespace: kserve-ci-e2e-test resourceVersion: "18045" uid: 5595ed62-198f-4022-bfe0-8a37d7214f0d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "18027" uid: d281ec05-7384-46eb-bc6e-9efee438dae2 kind: Event lastTimestamp: "2026-09-18T17:09:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:48Z" name: switch-graph-1ff44.18d67967dc8bc0d9 namespace: kserve-ci-e2e-test resourceVersion: "18179" uid: 879389df-4299-4948-89ac-87cc783a1b59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:09:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "18027" uid: d281ec05-7384-46eb-bc6e-9efee438dae2 kind: Event lastTimestamp: "2026-09-18T17:09:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T17:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:09:48Z" name: switch-graph-1ff44.18d67967dd184c9f namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: 04f268c1-185a-4776-bc7a-39774b89eb53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: d281ec05-7384-46eb-bc6e-9efee438dae2 kind: Event lastTimestamp: "2026-09-18T17:10:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:03Z" name: switch-graph-1ff44.18d6796b5c1b561d namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: af3945cb-ffc9-414f-82e5-a3e8a405e192 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:10:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1ff44 namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: d281ec05-7384-46eb-bc6e-9efee438dae2 kind: Event lastTimestamp: "2026-09-18T17:10:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T17:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:10:03Z" name: switch-graph-1ff44.18d6796b5cc6f3a3 namespace: kserve-ci-e2e-test resourceVersion: "18302" uid: 96cb24f2-7c78-4065-81ba-7f868fd89e2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T17:08:12.636472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a4cbd-58b99b6759-jv64t to ip-10-0-132-41.ec2.internal metadata: creationTimestamp: "2026-09-18T17:08:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T17:08:12Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d6795171e4badf namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: ee0a2b0e-97fe-4e5f-9f52-b9e3c86bacfa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dccbc56c5-lx4mq source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:13Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:13Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T17:08:13Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d67951940d8233 namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: 361d9dcf-3a0a-4447-940a-6a644e638b2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a4cbd} kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:13Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" metadata: creationTimestamp: "2026-09-18T17:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:13Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d6795195aba1b7 namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: 9df8a797-23b3-41f1-860d-8ec41c62d5c1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a4cbd} kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:af249ec9499da301cfba95a4607c7131eafdf4dd8f34e8ed6129fa8f55d0bc77" in 2.178s (2.178s including waiting). Image size: 211751504 bytes.' metadata: creationTimestamp: "2026-09-18T17:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:15Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d67952177fdbc1 namespace: kserve-ci-e2e-test resourceVersion: "16999" uid: 4dc95cc8-d6ea-4f46-b376-17e2d3b9152a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a4cbd} kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:15Z" message: 'Created container: switch-graph-a4cbd' metadata: creationTimestamp: "2026-09-18T17:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:15Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d679521ae0e671 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: b3387c72-ec84-4c95-98ec-9b5eb31aeba2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a4cbd} kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:15Z" message: Started container switch-graph-a4cbd metadata: creationTimestamp: "2026-09-18T17:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:15Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d679521b54d969 namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: f8d48fff-a632-4084-b55f-ef38a0c388b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a4cbd} kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:23Z" message: Stopping container switch-graph-a4cbd metadata: creationTimestamp: "2026-09-18T17:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:23Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d6795400596389 namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: 6908fb22-4bd5-4788-9c4e-4efc20e3c651 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a4cbd} kind: Pod name: switch-graph-a4cbd-58b99b6759-jv64t namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: a288cb1a-6b37-45ae-bcd1-221c0c86a113 kind: Event lastTimestamp: "2026-09-18T17:08:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:08:51Z" name: switch-graph-a4cbd-58b99b6759-jv64t.18d67954acabc2d9 namespace: kserve-ci-e2e-test resourceVersion: "17493" uid: 73946331-0373-45a4-8a52-822265abedcd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-41.ec2.internal source: component: kubelet host: ip-10-0-132-41.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a4cbd-58b99b6759 namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: b4262660-8fb9-4c09-ad06-99884a17bafd kind: Event lastTimestamp: "2026-09-18T17:08:12Z" message: 'Created pod: switch-graph-a4cbd-58b99b6759-jv64t' metadata: creationTimestamp: "2026-09-18T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:12Z" name: switch-graph-a4cbd-58b99b6759.18d679517141822c namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: 3d6968fc-55a0-426d-a287-74dad7a9c939 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a4cbd namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 17925664-9a5b-4870-a93b-e3adf86fa4cf kind: Event lastTimestamp: "2026-09-18T17:08:12Z" message: Scaled up replica set switch-graph-a4cbd-58b99b6759 from 0 to 1 metadata: creationTimestamp: "2026-09-18T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T17:08:12Z" name: switch-graph-a4cbd.18d679516f9d6310 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: 5e91f9ea-824b-4784-affc-1d6f18fe3ed4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:08:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a4cbd namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 575549e1-e76b-40ee-b656-c27ea991627e kind: Event lastTimestamp: "2026-09-18T17:08:12Z" message: InferenceGraph [switch-graph-a4cbd] is Ready metadata: creationTimestamp: "2026-09-18T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T17:08:12Z" name: switch-graph-a4cbd.18d6795176350dcf namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 5c022f6c-19cd-4c83-9503-0c3116495890 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "35748"