--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:20:15.546283Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-52e43-65c965b44d-tmkgc to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:20:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:20:15Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a20eb4e2b29 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: ce0520bb-1295-4235-9377-4da9de0bd15a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:15Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:20:15Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a2104d13cdc namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 0ac5abef-af72-4fd6-8a79-2043f4420c68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-52e43} kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:15Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a210611ae55 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: b0880b65-0f42-451b-9b0c-7d5c44c5847f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-52e43} kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: "2026-07-01T17:20:16Z" message: 'Created container: ensemble-graph-52e43' metadata: creationTimestamp: "2026-07-01T17:20:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:16Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a2109714e40 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 014aa115-7d05-4431-9055-bcae094cc945 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-52e43} kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: "2026-07-01T17:20:16Z" message: Started container ensemble-graph-52e43 metadata: creationTimestamp: "2026-07-01T17:20:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:16Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a2109edafc8 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: de69dc5d-f683-43b2-8d8c-b9e6776f1f23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-52e43} kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Stopping container ensemble-graph-52e43 metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:25Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a2343bcbd2f namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: 33e09dfa-5a43-4a7a-9092-841a588c2b85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:20:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-52e43} kind: Pod name: ensemble-graph-52e43-65c965b44d-tmkgc namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: c2c88393-d778-4166-aa58-f552479a37ba kind: Event lastTimestamp: "2026-07-01T17:20:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:53Z" name: ensemble-graph-52e43-65c965b44d-tmkgc.18be3a23f51611e3 namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 50f79dda-5bb8-409e-a3cc-e5092a450560 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-52e43-65c965b44d namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 35b9ff78-036f-44e8-adbc-5ebc04622d8d kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: 'Created pod: ensemble-graph-52e43-65c965b44d-tmkgc' metadata: creationTimestamp: "2026-07-01T17:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:15Z" name: ensemble-graph-52e43-65c965b44d.18be3a20eacde9b9 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: c95ab77e-b23d-48a2-9c0b-b376b5609769 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-52e43 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 32950931-955d-4c42-873f-715417aefe68 kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: Scaled up replica set ensemble-graph-52e43-65c965b44d from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:15Z" name: ensemble-graph-52e43.18be3a20e9e21808 namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 8391ae22-3d49-433f-b3d6-f2d982db90f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-52e43 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 24d834cc-6e17-40f0-a31e-c0d58c7a5436 kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: InferenceGraph [ensemble-graph-52e43] is Ready metadata: creationTimestamp: "2026-07-01T17:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:15Z" name: ensemble-graph-52e43.18be3a20f029af9c namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: b9e590b5-6ca4-4353-a213-5e5a896aa908 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:21:25.895217Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-bbcac-77d4868ccd-zlxf7 to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:21:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:21:25Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3a314c6b9820 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 95a4153c-cea4-419d-b57a-776377454db1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:21:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-bbcac-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:26Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3a3158e25030 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 4c64e23b-680a-446c-be7e-9cb067f67266 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:21:26Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:21:26Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3a318a326ddf namespace: kserve-ci-e2e-test resourceVersion: "21455" uid: c7fd12c6-052f-4fe9-8fa9-f9f64839fb59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bbcac} kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:21:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:26Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3a318bc3df57 namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 25e473ea-702b-4a1e-b919-c5028bc501b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bbcac} kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:21:27Z" message: 'Created container: ensemble-graph-bbcac' metadata: creationTimestamp: "2026-07-01T17:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:27Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3a318ea60445 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: d92bc5f4-f4ae-4004-9088-fecb9f6111f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bbcac} kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:21:27Z" message: Started container ensemble-graph-bbcac metadata: creationTimestamp: "2026-07-01T17:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:27Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3a318f19f98a namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: ca382b9b-7b21-464b-bafc-c366884e736b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bbcac} kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Stopping container ensemble-graph-bbcac metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:40Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3aa472230ca9 namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: a7350ec3-31b0-4b3a-bff6-c9acb0ce353a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:29:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bbcac} kind: Pod name: ensemble-graph-bbcac-77d4868ccd-zlxf7 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 9827710e-da00-4730-af3d-3bbe29d57231 kind: Event lastTimestamp: "2026-07-01T17:30:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:09Z" name: ensemble-graph-bbcac-77d4868ccd-zlxf7.18be3aa57c6d5356 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 6188efd1-b430-4aea-ae0f-719b394e1e94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-bbcac-77d4868ccd namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: cbdb9f98-1148-4899-9449-268650f42c61 kind: Event lastTimestamp: "2026-07-01T17:21:25Z" message: 'Created pod: ensemble-graph-bbcac-77d4868ccd-zlxf7' metadata: creationTimestamp: "2026-07-01T17:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:25Z" name: ensemble-graph-bbcac-77d4868ccd.18be3a314be0585c namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: 8121d111-5eab-4281-baf1-a648b845fc3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: fe072c73-80ce-4d31-a57a-6cfd6b338d23 kind: Event lastTimestamp: "2026-07-01T17:21:25Z" message: Scaled up replica set ensemble-graph-bbcac-77d4868ccd from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:25Z" name: ensemble-graph-bbcac.18be3a314a13653a namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 35fb1218-c9b3-4442-84d5-fdba73d2abca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: 6f5252e6-512e-49e3-929e-f67ab1496ac7 kind: Event lastTimestamp: "2026-07-01T17:21:25Z" message: InferenceGraph [ensemble-graph-bbcac] is Ready metadata: creationTimestamp: "2026-07-01T17:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:25Z" name: ensemble-graph-bbcac.18be3a31500f9b51 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: a6ad1351-5b30-4f96-aa88-4ee1bddce747 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 6f5252e6-512e-49e3-929e-f67ab1496ac7 kind: Event lastTimestamp: "2026-07-01T17:21:26Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-bbcac": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-bbcac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:26Z" name: ensemble-graph-bbcac.18be3a3152dfb60a namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: c83aa159-2d52-4dd0-b362-0543727cf877 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 6f5252e6-512e-49e3-929e-f67ab1496ac7 kind: Event lastTimestamp: "2026-07-01T17:21:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-bbcac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:26Z" name: ensemble-graph-bbcac.18be3a3152e008dd namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: e93dfdd2-cbd1-47bb-a291-3d285ba40e26 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 488b819c-46dd-4ca7-a827-5777f4c354c5 kind: Event lastTimestamp: "2026-07-01T17:22:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:10Z" name: ensemble-graph-bbcac.18be3a34c8aaec72 namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: 60316f95-9aac-4cc2-8220-afe2bc07b551 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 488b819c-46dd-4ca7-a827-5777f4c354c5 kind: Event lastTimestamp: "2026-07-01T17:22:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:10Z" name: ensemble-graph-bbcac.18be3a34c912dc3c namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: a0df35f2-d47b-416a-8c53-4881e017ff8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:08:06.700248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:08:06Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397738b03874 namespace: kserve-ci-e2e-test resourceVersion: "14165" uid: 05ce8087-c383-4e0e-ade1-e18e6b21c913 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1b15a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:06Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397746ef12bb namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 06e0627c-38af-4042-bc84-dbf436e66302 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:08:07Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be3977761d7c59 namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: e8d85b5a-8b94-416d-85fd-d31b0f5250d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:07Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be39777848d442 namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 758cc0db-29ed-421a-9a47-601d644da90c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" in 17.641s (17.641s including waiting). Image size: 1334626409 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397b93c9e206 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 4d9eb8f3-f9bd-4a9c-9cd4-7639ecfd60f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397b978f2eda namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 41deb300-bd93-4a1f-b1c1-a0f6e7c3499b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397b9813b566 namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: a3ac0948-130c-4079-9e5d-3cda8ab11449 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397b982135fc namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 6a74d432-498d-497c-a839-d1cda4963d4f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.991s (2.991s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397c4a72568b namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: b376e54b-bf6b-467f-8b15-356536ae452a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397c4d8c81f4 namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 395dff2d-1388-40bf-b3d7-c34f8afcc325 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397c4e06614e namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: 1e268dc7-0aad-44d5-b066-b832ffab7103 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:09:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:05Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be397c70526781 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 452cfc9a-699e-4aa2-b0d4-cf3659ca5bcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be398f7250ae94 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 54c9b393-1295-48f7-b1ad-52e341d001fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 22a5235c-0a90-4a58-b636-3c53080e34c7 kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx.18be398f7250fe87 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 214be86c-5cff-4d5c-a813-5f83337bfca6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1b15a-predictor-5c76b7cdd8 namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: 34d15913-2c27-4ee1-8e46-7e19aceade47 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'Created pod: error-404-isvc-1b15a-predictor-5c76b7cdd8-56vrx' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: error-404-isvc-1b15a-predictor-5c76b7cdd8.18be397737dcaf2a namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: 27e47da1-677c-4874-94b4-c400377382bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: cfe7407b-ef27-4334-a52a-8a42e7e55c99 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Scaled up replica set error-404-isvc-1b15a-predictor-5c76b7cdd8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: error-404-isvc-1b15a-predictor.18be3977347b4341 namespace: kserve-ci-e2e-test resourceVersion: "14153" uid: 9167c606-3113-4f5f-bc2c-5aeb301b6a84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: 4c8b7177-b7a5-413a-8021-fb084637da60 kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: error-404-isvc-1b15a-predictor.18be397ab3f63bff namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: d0e76be3-339b-4aa8-8422-6baef58a8ddb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: 4c8b7177-b7a5-413a-8021-fb084637da60 kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: error-404-isvc-1b15a-predictor.18be397ab4ab7220 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 4b5a87ce-d08b-4e52-bd29-93af30eb90d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 4c8b7177-b7a5-413a-8021-fb084637da60 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:36Z" name: error-404-isvc-1b15a-predictor.18be3981b2fafd83 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 665ea907-af71-4018-8008-60c069de36c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 4c8b7177-b7a5-413a-8021-fb084637da60 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:36Z" name: error-404-isvc-1b15a-predictor.18be3981b38d3629 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: ac09d7f0-1bbb-43f8-82d1-05d33e30c05b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1b15a namespace: kserve-ci-e2e-test resourceVersion: "14135" uid: 379e3345-0be5-4632-ae04-f2d83af9e19b kind: Event lastTimestamp: "2026-07-01T17:09:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:15Z" name: error-404-isvc-1b15a.18be397735476ec8 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: c7f16ff5-0171-41cf-a2be-60187b5c604c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1b15a namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 379e3345-0be5-4632-ae04-f2d83af9e19b kind: Event lastTimestamp: "2026-07-01T17:09:15Z" message: InferenceService [error-404-isvc-1b15a] is Ready metadata: creationTimestamp: "2026-07-01T17:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:15Z" name: error-404-isvc-1b15a.18be39872bbd3e55 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 6e7ad349-c0e1-43b7-9723-820a52e53202 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:19:31.491642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16a96fa58f namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: c8c15313-028d-4077-8a6e-e33819546aab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1f194-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16b1aa436d namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 3118ad17-5adf-4ae0-99f6-43b97d4a7016 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16e7cc4570 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 2849797e-ccb6-491d-b058-4c0df0955c9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16e93e801d namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: ba502805-1cc2-434f-8f24-ad7f8578f924 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16ebd90bb4 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 797d810c-8931-4959-a087-dedb08859dd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16ec6c29a2 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 630cee45-6706-4238-a4e9-870aabfeaac9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16ec7b75ef namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: 54aac7f6-b068-41fd-ad3c-577a830f6f05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16ef644fda namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 8a0750c7-f124-4d2a-8849-fb94d2d9e6d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16efd9a8dc namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: cd9b09a8-e0d9-45af-b45c-e67079ded885 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:20:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:18Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a16f70d7539 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 1aedec3a-385c-4f92-a378-232b0adde94e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a294d739ba2 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: c22c0b88-8ce3-45d2-b13f-7fcc318f5a13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a294d7431e3 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 308f6f6f-ab1a-4d1e-a83d-14c76203761c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: dc2febf1-bc84-4180-8f3c-da46de283e1b kind: Event lastTimestamp: "2026-07-01T17:20:53Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:53Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz.18be3a29d3534694 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 786bf0f5-8575-4715-93de-b7a4dc155067 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1f194-predictor-6c5d4fc46b namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 73035fc5-86a7-4a1e-ab02-3b96f38656ec kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: 'Created pod: error-404-isvc-1f194-predictor-6c5d4fc46b-kk5sz' metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-1f194-predictor-6c5d4fc46b.18be3a16a8e5901f namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: 76514986-3c9e-4b83-aec9-df6efada465b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "19985" uid: 4e188558-7029-4a55-b53e-d698ed569186 kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Scaled up replica set error-404-isvc-1f194-predictor-6c5d4fc46b from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-1f194-predictor.18be3a16a70f1637 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 6768f20e-0254-4891-9170-5e630b135767 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:19:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 3bd3fa94-f077-4ba5-9626-eb1e6007affd kind: Event lastTimestamp: "2026-07-01T17:20:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:16Z" name: error-404-isvc-1f194-predictor.18be3a1a2594a146 namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 5dc2bb0b-eb90-4689-bd34-be13a26de4c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:19:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 3bd3fa94-f077-4ba5-9626-eb1e6007affd kind: Event lastTimestamp: "2026-07-01T17:20:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:16Z" name: error-404-isvc-1f194-predictor.18be3a1a260de8ba namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 835a0373-8d28-421e-89d0-1117fac1c707 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 3bd3fa94-f077-4ba5-9626-eb1e6007affd kind: Event lastTimestamp: "2026-07-01T17:20:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:46Z" name: error-404-isvc-1f194-predictor.18be3a1da4bb386b namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 33a9dc68-c5fe-425a-8b6e-436c0b9e565d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 3bd3fa94-f077-4ba5-9626-eb1e6007affd kind: Event lastTimestamp: "2026-07-01T17:20:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:46Z" name: error-404-isvc-1f194-predictor.18be3a1da52d3bdb namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 49d8ea01-51b1-4a33-b8a2-d748ec9f8a56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f194 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 936ab739-2abc-46b7-b50d-7b1515d2b78b kind: Event lastTimestamp: "2026-07-01T17:20:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:29Z" name: error-404-isvc-1f194.18be3a16a71c2e71 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 18dc8598-dd65-4e72-9f9c-a479f12e0d93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f194 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 936ab739-2abc-46b7-b50d-7b1515d2b78b kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-1f194": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f194": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-1f194.18be3a16a78b6ced namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 1de30146-abd9-4c09-843f-b17a6d1b75c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f194 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 936ab739-2abc-46b7-b50d-7b1515d2b78b kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f194": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-1f194.18be3a16a78bbc2b namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: e39a7db4-ad6b-4a31-8337-39d71aba0797 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f194 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: 936ab739-2abc-46b7-b50d-7b1515d2b78b kind: Event lastTimestamp: "2026-07-01T17:20:28Z" message: InferenceService [error-404-isvc-1f194] is Ready metadata: creationTimestamp: "2026-07-01T17:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:28Z" name: error-404-isvc-1f194.18be3a2409bbce99 namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: ed17d186-e592-4fb7-9655-70c7fb1db1fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:09:50.674512Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3c336-predictor-54665d447b-s2vc2 to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f6e098572 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: ca1d4c21-8b9e-4742-aa72-7d26299f484a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f87e7ee18 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 1d870888-407c-459b-9634-3fff4c9e8578 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f896ee746 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: ab877fde-a91c-4e3c-b951-586ae3826f66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f8c586a62 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 28fe2793-0ce3-4e5c-9fc3-8e83c42b9066 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f8ccf1f54 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 9d9fa498-5894-491e-b0d4-db6bc080839f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f8cd92698 namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: b43061f0-b0fb-410c-b8e0-228572623d80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f8fea0333 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: e62bde45-33eb-4997-92a5-af7e29bf1667 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f90636933 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: a376c68e-e2c3-4116-b73e-b779cb89130c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:19:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:17Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be398f9b03d0bd namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: 7d056584-c650-4dd2-873b-662e90a7d443 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be3a12fa0178d4 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 0ae699e8-2ce8-4164-a5c1-b2afd05126e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be3a12fa02ee96 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 0100abd5-8135-45ca-baa2-674d4ed6f85d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c336-predictor-54665d447b-s2vc2 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 0ce82ad2-7236-475b-b3ff-eccfa49eccfa kind: Event lastTimestamp: "2026-07-01T17:19:17Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:17Z" name: error-404-isvc-3c336-predictor-54665d447b-s2vc2.18be3a1363c511b3 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 0ec1af67-8764-4d35-8166-cc3350649e4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3c336-predictor-54665d447b namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 6aed56fe-d7e2-4165-a9fd-6c4626566ebd kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'Created pod: error-404-isvc-3c336-predictor-54665d447b-s2vc2' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-3c336-predictor-54665d447b.18be398f6d5c8cf5 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: de7ee5ac-9b8f-41ac-8951-0f5daf7b2a99 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: d7eadefa-2250-4a12-8e14-bb7e87955ab6 kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Scaled up replica set error-404-isvc-3c336-predictor-54665d447b from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-3c336-predictor.18be398f6b8d7cc0 namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: bbf6b9ba-30d2-4912-87d4-1e5b3b24663e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 2976700f-c261-4396-84a0-1b33f8211e4d kind: Event lastTimestamp: "2026-07-01T17:10:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:20Z" name: error-404-isvc-3c336-predictor.18be3992ea419538 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: cf643d25-8234-4860-b26f-469554a96971 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 2976700f-c261-4396-84a0-1b33f8211e4d kind: Event lastTimestamp: "2026-07-01T17:10:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:20Z" name: error-404-isvc-3c336-predictor.18be3992eaaf4275 namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 88de13c2-df7d-492e-8138-99119dc37023 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 2976700f-c261-4396-84a0-1b33f8211e4d kind: Event lastTimestamp: "2026-07-01T17:11:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:20Z" name: error-404-isvc-3c336-predictor.18be3999e8632254 namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 602a8921-0acf-4408-bad6-0fc469b62109 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 2976700f-c261-4396-84a0-1b33f8211e4d kind: Event lastTimestamp: "2026-07-01T17:11:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:20Z" name: error-404-isvc-3c336-predictor.18be3999e9450f25 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 85ef0920-28ae-4549-979c-91482c03a148 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: b38a5d21-ce1a-4fc9-b865-def498999b6c kind: Event lastTimestamp: "2026-07-01T17:10:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:37Z" name: error-404-isvc-3c336.18be398f6bd4f4b4 namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 14106061-cc3c-4b21-b001-b81fa27ea055 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: b38a5d21-ce1a-4fc9-b865-def498999b6c kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-3c336": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3c336": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-3c336.18be398f6c8c2d11 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: e05c387c-0cd0-4d7b-9c6f-720639eeb26b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: b38a5d21-ce1a-4fc9-b865-def498999b6c kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3c336": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:50Z" name: error-404-isvc-3c336.18be398f6c8c6512 namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: ecb2c9f1-4c56-4b64-85a6-27d4ebcba574 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: b38a5d21-ce1a-4fc9-b865-def498999b6c kind: Event lastTimestamp: "2026-07-01T17:10:37Z" message: InferenceService [error-404-isvc-3c336] is Ready metadata: creationTimestamp: "2026-07-01T17:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:37Z" name: error-404-isvc-3c336.18be399a56bec927 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 8d14ec43-ddbf-4659-b639-67b83dc97b39 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:40.701697Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4517b-predictor-677799f794-kj8t9 to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4812f4056 namespace: kserve-ci-e2e-test resourceVersion: "24983" uid: dc1c3610-f369-4890-91b5-2d3b0d38da3d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4517b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa48f1e03da namespace: kserve-ci-e2e-test resourceVersion: "25011" uid: 3b0a695d-bb1b-436e-9d9a-ee90116c0081 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24997" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4be624172 namespace: kserve-ci-e2e-test resourceVersion: "25033" uid: fd736061-24c5-40b7-a566-ba85715218cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4c0035a13 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: fd8182ad-6e69-4c4f-a248-0a53e27e17f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4c321227a namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: a3511d6a-d3b3-473e-8956-e6ae56c8c08b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4c39a84a9 namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: 9513f729-ed71-4cab-bdf0-9a87fa553ffe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4c3a6d81b namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 07011b9d-d1b9-49ab-b999-478fd3aa45f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4c63a1e4a namespace: kserve-ci-e2e-test resourceVersion: "25039" uid: d0323ecb-b0db-4077-8261-8c43d0456d80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa4c6b1ec24 namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: c045478d-0d8c-45da-9d2b-e0eb6e677fee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:29:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:30:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:29Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3aa51bec15a6 namespace: kserve-ci-e2e-test resourceVersion: "25858" uid: 9a25e19d-e95b-4e1d-9b6a-6d19a10a4084 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3ab73b401ed7 namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: e319c342-6951-4a81-9117-7ad00667e208 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3ab73b40d920 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: c91af1f4-bf51-4d63-a57e-37a3a8122ff2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4517b-predictor-677799f794-kj8t9 namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 73bda8d8-398f-46d6-a526-be667395fad7 kind: Event lastTimestamp: "2026-07-01T17:31:04Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:04Z" name: error-404-isvc-4517b-predictor-677799f794-kj8t9.18be3ab7f83df6b3 namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: b296b33f-ba1d-4ecd-918f-acbd5eea9c12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4517b-predictor-677799f794 namespace: kserve-ci-e2e-test resourceVersion: "24971" uid: dc284a09-d3df-4f3d-aff8-b4283bc0fcb1 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'Created pod: error-404-isvc-4517b-predictor-677799f794-kj8t9' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-4517b-predictor-677799f794.18be3aa4809b56f8 namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: af7834d1-6b55-4479-b00b-7fd84ed31788 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24968" uid: 5bf5669d-98d2-4720-b7c4-65e4973bfd4f kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Scaled up replica set error-404-isvc-4517b-predictor-677799f794 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-4517b-predictor.18be3aa47f036769 namespace: kserve-ci-e2e-test resourceVersion: "24973" uid: aa2431f3-75f2-4510-959f-093108be82fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: af954d29-c3e4-4362-8d27-6427ed03c949 kind: Event lastTimestamp: "2026-07-01T17:30:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:10Z" name: error-404-isvc-4517b-predictor.18be3aa7fe0ccc1d namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 038b874d-ae03-4bdd-a9a1-f561bc158f27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: af954d29-c3e4-4362-8d27-6427ed03c949 kind: Event lastTimestamp: "2026-07-01T17:30:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:10Z" name: error-404-isvc-4517b-predictor.18be3aa7fe766757 namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 884b7ebe-5fab-4bcc-a3ee-ee39fa111b11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25167" uid: af954d29-c3e4-4362-8d27-6427ed03c949 kind: Event lastTimestamp: "2026-07-01T17:30:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:55Z" name: error-404-isvc-4517b-predictor.18be3aaefbdab402 namespace: kserve-ci-e2e-test resourceVersion: "26145" uid: dec2807a-3cb4-4d40-91e1-1f2252c3c8a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25167" uid: af954d29-c3e4-4362-8d27-6427ed03c949 kind: Event lastTimestamp: "2026-07-01T17:30:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:55Z" name: error-404-isvc-4517b-predictor.18be3aaefc4b6b3f namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: 85e3b03a-a2b1-4a4a-a955-454b962635d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 7c750621-9b84-44fe-9824-7827156e3c3b kind: Event lastTimestamp: "2026-07-01T17:30:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:39Z" name: error-404-isvc-4517b.18be3aa47f4b34c8 namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: a12c61ff-8764-4032-9a14-2251ed9ce6ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 7c750621-9b84-44fe-9824-7827156e3c3b kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-4517b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4517b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-4517b.18be3aa47fa015d5 namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: ea7520b0-c44c-449b-9e25-b6a8554c87ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 7c750621-9b84-44fe-9824-7827156e3c3b kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4517b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-4517b.18be3aa47fa05b66 namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: ea8c5d01-fb67-4822-87ce-075b094c18b9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 7c750621-9b84-44fe-9824-7827156e3c3b kind: Event lastTimestamp: "2026-07-01T17:30:39Z" message: InferenceService [error-404-isvc-4517b] is Ready metadata: creationTimestamp: "2026-07-01T17:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:39Z" name: error-404-isvc-4517b.18be3ab22b8452d0 namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 622970d7-55cb-4b85-b2da-0fd3653da471 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:19:15.601021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-52e43-predictor-79768fc67d-xk27k to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a12f647ef82 namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 3e12d296-c4e2-4699-bc9e-4081c8e02a93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a131021c3dc namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: 76eddecc-97fb-497b-81d9-d492eeacfcd8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a131190a2fd namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: 987b3738-1a18-43e9-b775-00290546a377 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a1314690a28 namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 8346f9bb-ae5e-4a68-a9c5-cfca4ce50fce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a1314e5118b namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: d9defa95-7848-4655-99a6-bf604142ec15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a1314f07bf8 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 572dab2a-9842-4380-9241-8cf21a10931c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a1317edc918 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 19105feb-0515-4816-9601-36d28dd9bfd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a1318652415 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 1ee02c91-547c-4377-a278-360cb5c241c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:20:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:03Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a1374705d0b namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: aaa782e6-e3c8-4785-a3f9-af23518b1bdb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a2357fcb016 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 58d46791-6512-472d-9d46-e360685efdf4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a2357ff9986 namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: d51a1f72-d5e4-42d8-ad27-5cd2b74dae0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52e43-predictor-79768fc67d-xk27k namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: e49336bc-9319-467b-a6b8-969be03815e0 kind: Event lastTimestamp: "2026-07-01T17:20:28Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:28Z" name: error-404-isvc-52e43-predictor-79768fc67d-xk27k.18be3a23fc93a807 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 65fcbb91-44d4-4910-88fa-1184789e69fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-52e43-predictor-79768fc67d namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: 882b6588-e56d-4e61-9881-728da963babb kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'Created pod: error-404-isvc-52e43-predictor-79768fc67d-xk27k' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-52e43-predictor-79768fc67d.18be3a12f5e392dc namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 850e75a7-b425-4608-85b5-03d030842f1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: 29a66497-5106-4a56-847c-5180d9b5ddaf kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Scaled up replica set error-404-isvc-52e43-predictor-79768fc67d from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-52e43-predictor.18be3a12f3c2f722 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: b6612b7e-7021-4e2e-aa56-de0c9a76aed2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: b6ec2d31-9ac1-49be-8007-8aeae3caf5dd kind: Event lastTimestamp: "2026-07-01T17:19:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:30Z" name: error-404-isvc-52e43-predictor.18be3a1672295f07 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: 77ee5264-0747-442e-bfa0-c2799f1eeb40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: b6ec2d31-9ac1-49be-8007-8aeae3caf5dd kind: Event lastTimestamp: "2026-07-01T17:19:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:30Z" name: error-404-isvc-52e43-predictor.18be3a16729af6e1 namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 94056a6d-706b-4f9a-972c-c2db8c8c080b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:19:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: b6ec2d31-9ac1-49be-8007-8aeae3caf5dd kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:15Z" name: error-404-isvc-52e43-predictor.18be3a19f1553bdc namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: c4e66d18-1d7f-4d6f-8128-fa2fdb5e047c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:19:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: b6ec2d31-9ac1-49be-8007-8aeae3caf5dd kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:15Z" name: error-404-isvc-52e43-predictor.18be3a19f1c2059b namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: d94bbac3-5628-4674-82ec-0cf6859fda60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 7f35b083-18d8-4edf-8091-0c2aedb5b8f0 kind: Event lastTimestamp: "2026-07-01T17:20:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:13Z" name: error-404-isvc-52e43.18be3a12f3b5f466 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: 628f8a9f-ebab-4a73-9c7e-8c29114257ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 7f35b083-18d8-4edf-8091-0c2aedb5b8f0 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-52e43": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-52e43": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-52e43.18be3a12f42cbff2 namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: 6922a0d8-68ee-4d1a-aa7c-a3ceb6c8d97e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 7f35b083-18d8-4edf-8091-0c2aedb5b8f0 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-52e43": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:15Z" name: error-404-isvc-52e43.18be3a12f42d008b namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: 95bb95fe-0d47-43b0-9b0d-882cba8c4582 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 7f35b083-18d8-4edf-8091-0c2aedb5b8f0 kind: Event lastTimestamp: "2026-07-01T17:20:13Z" message: InferenceService [error-404-isvc-52e43] is Ready metadata: creationTimestamp: "2026-07-01T17:20:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:13Z" name: error-404-isvc-52e43.18be3a2083faf317 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 1433fbff-0a73-4412-81d4-af1c26e323d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:20:51.734838Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a29584da4d0 namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: 62349161-b93e-4217-afd7-fa623ef7f974 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a2972485389 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 3af5fcf6-05be-4706-be2a-3d40446f5bb1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a2973aec765 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: aa746cc3-5d75-46a8-92fd-3c80f82acbb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a29767d9bef namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: a915b625-f875-4e86-b25d-479b62b6b5e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a2976f39af3 namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 031ad6ce-d889-478f-9a9b-2d45a0d16abd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a2976fe5d5c namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: 7e37e601-4f2d-4f9d-ae6d-a75c5e78c1d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a29796fe668 namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 6554d591-5080-43d9-b89a-016d4e3f620e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a2979e834a3 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: fe93a3f6-7c84-47a3-84d3-32e18c7d802a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:30:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:10Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3a29e9be6c8c namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 510bca88-4600-4254-8399-6ddf357a5a37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3aaa7a6f315f namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: e1346b34-d632-43f1-9743-7fd3a8b25aa6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3aaa7a6fe31c namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: a995a7ab-1eec-468c-aec0-e52057e6545d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: c6b534f1-c10d-494c-8547-05f4a1073b40 kind: Event lastTimestamp: "2026-07-01T17:30:10Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:10Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d.18be3aab5e56a8cd namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: 8f78f06f-32ca-411b-bbce-67392d928e3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6460c-predictor-5d5d94dfb6 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 9796f492-e48d-4bf8-a0bc-4efa0410404a kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: 'Created pod: error-404-isvc-6460c-predictor-5d5d94dfb6-btq8d' metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-6460c-predictor-5d5d94dfb6.18be3a2957e4a088 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 2856073b-abea-4cd9-a188-1a84b49f2b22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 64d93716-9d4d-4b98-8331-0a77d870b8b1 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Scaled up replica set error-404-isvc-6460c-predictor-5d5d94dfb6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-6460c-predictor.18be3a29560e2f59 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 734523bd-edfd-46e7-85f4-c30695d38ea0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 23cd8d1c-b583-48db-a4ea-365e844f7e02 kind: Event lastTimestamp: "2026-07-01T17:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:21Z" name: error-404-isvc-6460c-predictor.18be3a2cd491ef0c namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 8fe106bf-cb72-43a7-9dd6-67993110cb1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 23cd8d1c-b583-48db-a4ea-365e844f7e02 kind: Event lastTimestamp: "2026-07-01T17:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:21Z" name: error-404-isvc-6460c-predictor.18be3a2cd5021083 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: 49da753e-e96a-4785-b8ac-9cdb4eedf90f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 23cd8d1c-b583-48db-a4ea-365e844f7e02 kind: Event lastTimestamp: "2026-07-01T17:22:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:06Z" name: error-404-isvc-6460c-predictor.18be3a33d2ada606 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 77dce4e1-9a01-4626-8b72-2bf4f9d7a9e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 23cd8d1c-b583-48db-a4ea-365e844f7e02 kind: Event lastTimestamp: "2026-07-01T17:22:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:06Z" name: error-404-isvc-6460c-predictor.18be3a33d3184fec namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 6e2fdde0-738d-4504-8255-7a769ec98899 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6460c namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 0dcc6a65-1711-4cea-9080-379c07cd5888 kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:40Z" name: error-404-isvc-6460c.18be3a29563f3395 namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 1c9763fd-8834-4afc-a1ab-37a71b754f79 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6460c namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 0dcc6a65-1711-4cea-9080-379c07cd5888 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-6460c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6460c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-6460c.18be3a2956bc0599 namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 6083b941-1297-4e0c-ae74-946c040dbe05 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6460c namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 0dcc6a65-1711-4cea-9080-379c07cd5888 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6460c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:51Z" name: error-404-isvc-6460c.18be3a2956bc5d49 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 0811bd3c-5fa1-4660-828a-88ba5004df55 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6460c namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 0dcc6a65-1711-4cea-9080-379c07cd5888 kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: InferenceService [error-404-isvc-6460c] is Ready metadata: creationTimestamp: "2026-07-01T17:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:40Z" name: error-404-isvc-6460c.18be3a34a98cd7f5 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: dd3cc6eb-4376-4e29-8248-a231f133272b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:20:25.898683Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a23545930dd namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 1c50bb7e-7d55-4010-b72f-8549027c8327 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-bbcac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a236109c3a8 namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: 234f2829-fbd9-4181-9d87-764c3e212067 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:20:26Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a239285f7a9 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: ccb50b96-6b29-4a7e-a409-95f240ba9d5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a2393edd04e namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 8acac73a-0052-45b1-9fa6-cfd97e202ca3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:27Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a2396d7b5f7 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: e7ca3b45-1431-452e-9ae4-4b31489babc5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:27Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a23975709ed namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: 257a4588-9e26-44a4-bed8-8efa227f8a2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:27Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a23976128f1 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 9c28d634-97d6-46e4-8149-ec6e89df73fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:27Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a239a19ebfa namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 5f074745-267e-4ed9-8597-4ab3c451882d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:20:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:27Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a239a945533 namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: b072b565-ca02-4b6e-b327-cbde9e83d1a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:20:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:21:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:05Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3a23d4aac025 namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 9670b825-5579-4129-8428-5bcef4989b65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3aa4840d04e5 namespace: kserve-ci-e2e-test resourceVersion: "25003" uid: bfbce358-496a-4883-ba1d-418381a3c9b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9b4cccca-1a19-448a-b584-acf00f272af2 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:40Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h.18be3aa4840da079 namespace: kserve-ci-e2e-test resourceVersion: "25005" uid: 16b6e963-6fee-43be-8fdc-02e405b58d11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bbcac-predictor-5ff4fd5cc4 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 34863dea-774c-4edd-a257-ea531f2a2286 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: 'Created pod: error-404-isvc-bbcac-predictor-5ff4fd5cc4-wtk7h' metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-bbcac-predictor-5ff4fd5cc4.18be3a2353c179cd namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 022bd711-ef40-4943-a041-92e139db4fd6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 4e6aede8-9b85-40ea-bec6-74dc0fcde586 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Scaled up replica set error-404-isvc-bbcac-predictor-5ff4fd5cc4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-bbcac-predictor.18be3a2351ac1fd7 namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: 65f07cc8-e006-4ac1-930f-bd756a7697a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:20:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: 0a81b87c-00e7-4387-8286-ed6fd40e699c kind: Event lastTimestamp: "2026-07-01T17:20:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:55Z" name: error-404-isvc-bbcac-predictor.18be3a26d0345476 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 0f794772-a056-4fc3-b4a0-c17ce2e47f81 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:20:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: 0a81b87c-00e7-4387-8286-ed6fd40e699c kind: Event lastTimestamp: "2026-07-01T17:20:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:55Z" name: error-404-isvc-bbcac-predictor.18be3a26d0a4b6bd namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: ce7fc6e0-e725-445c-ac28-96a11362d445 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 0a81b87c-00e7-4387-8286-ed6fd40e699c kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:40Z" name: error-404-isvc-bbcac-predictor.18be3a2dce142678 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: 204f41ea-03cf-4aef-a084-0747f7808711 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 0a81b87c-00e7-4387-8286-ed6fd40e699c kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:40Z" name: error-404-isvc-bbcac-predictor.18be3a2dce5f97ba namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: d33688dd-3cc8-4a2d-9327-0c85e527e910 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbcac namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: e5ce886d-d862-4278-8f3b-7640c86b43a6 kind: Event lastTimestamp: "2026-07-01T17:21:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:15Z" name: error-404-isvc-bbcac.18be3a2351ac9407 namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: a4dd1451-ca4d-4342-8b37-6515b748671e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbcac namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: e5ce886d-d862-4278-8f3b-7640c86b43a6 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-bbcac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bbcac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-bbcac.18be3a23526a9ea1 namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: 25a942a1-fff5-4de6-9b70-d1f43a3e4a9a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbcac namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: e5ce886d-d862-4278-8f3b-7640c86b43a6 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bbcac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:25Z" name: error-404-isvc-bbcac.18be3a23526aed64 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 91fd069e-b3dc-4330-8b87-1f97de916638 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: e5ce886d-d862-4278-8f3b-7640c86b43a6 kind: Event lastTimestamp: "2026-07-01T17:21:15Z" message: InferenceService [error-404-isvc-bbcac] is Ready metadata: creationTimestamp: "2026-07-01T17:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:15Z" name: error-404-isvc-bbcac.18be3a2ecc5be132 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 11f437ac-b009-4cd5-9b85-66685d0136eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:31:01.060379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab736efa349 namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: 0fc3f97a-41d6-4e5a-98b1-073aa82b2c70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-bbf0f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab74785ad66 namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 7dcf574c-4e56-40c0-b110-6d4f63ae4430 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab77571016f namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: 09cd1d81-1081-431a-adf2-fcf7b7238f49 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab776d95aac namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: 4cb59eac-539f-4921-b88b-ccf123000ff6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab779b90440 namespace: kserve-ci-e2e-test resourceVersion: "26324" uid: 0680b2ec-be12-4f8b-8ef3-4aad2af1def6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab77a2dacc0 namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: 1d2a9b5e-de15-4b0d-beaa-563765ccf772 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab77a376961 namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: d648e54e-eda0-4f1d-8c3a-7b8c740e82c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab77da16cc1 namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: 3853f984-e215-4a60-825c-edb824aa0543 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab77e1cf1c7 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: d9bb2ad2-3275-49d4-aefa-c39fa6bfd08d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:40:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:18Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3ab79650dfec namespace: kserve-ci-e2e-test resourceVersion: "30398" uid: 062974c4-279b-4500-b47b-7963e9e1c597 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3b38624108dd namespace: kserve-ci-e2e-test resourceVersion: "30371" uid: 41d2531c-1dc3-4965-b02d-45540a390ade reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3b3862418e2d namespace: kserve-ci-e2e-test resourceVersion: "30372" uid: 09319860-0f55-42db-acce-82b545809963 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 3121a651-3759-41c0-8363-d5cf6fbb991a kind: Event lastTimestamp: "2026-07-01T17:40:18Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:18Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn.18be3b390abfcc17 namespace: kserve-ci-e2e-test resourceVersion: "30396" uid: 7cc0cbe0-ce93-4fd5-91f1-740fd6f1e03e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bbf0f-predictor-85ff6cdb4 namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: c203af47-057b-4567-9bc9-33097220ca4f kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: 'Created pod: error-404-isvc-bbf0f-predictor-85ff6cdb4-jzkmn' metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-bbf0f-predictor-85ff6cdb4.18be3ab7365ee1fe namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: 8eb6355f-3b7f-449c-83c6-fe475f1b6f09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: 9e2eeee6-12cb-40f8-9ff9-ad1ddaeec523 kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: Scaled up replica set error-404-isvc-bbf0f-predictor-85ff6cdb4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-bbf0f-predictor.18be3ab734562451 namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: 3e8e885c-05f2-40e4-a1b5-c7e1fdbae972 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:31:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 952f002a-25db-4a44-8b21-4fb63caebffc kind: Event lastTimestamp: "2026-07-01T17:31:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:46Z" name: error-404-isvc-bbf0f-predictor.18be3abab2a58e79 namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: a82e83b8-3e72-4e3c-b457-df2dcdafe249 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:31:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 952f002a-25db-4a44-8b21-4fb63caebffc kind: Event lastTimestamp: "2026-07-01T17:31:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:46Z" name: error-404-isvc-bbf0f-predictor.18be3abab31f87c6 namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 4d0ce435-8222-4234-a909-2cc9216b25e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:31:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 952f002a-25db-4a44-8b21-4fb63caebffc kind: Event lastTimestamp: "2026-07-01T17:32:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:31Z" name: error-404-isvc-bbf0f-predictor.18be3abe31cec9e0 namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: a4629813-24ac-4581-a110-048fd5aa86eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:31:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 952f002a-25db-4a44-8b21-4fb63caebffc kind: Event lastTimestamp: "2026-07-01T17:32:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:31Z" name: error-404-isvc-bbf0f-predictor.18be3abe32333c85 namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 0f51f2a6-2a34-4973-a8c3-25f5da68b1d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 6c7f385d-ac39-46ca-9e91-7c7319e2458a kind: Event lastTimestamp: "2026-07-01T17:31:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:48Z" name: error-404-isvc-bbf0f.18be3ab734183d02 namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: e1ea028d-4816-4f5f-bbee-2423daf59512 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 6c7f385d-ac39-46ca-9e91-7c7319e2458a kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-bbf0f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bbf0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-bbf0f.18be3ab7347443ee namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: 5768f8c8-a2d9-424a-8bea-6f283898b53a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 6c7f385d-ac39-46ca-9e91-7c7319e2458a kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bbf0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:01Z" name: error-404-isvc-bbf0f.18be3ab734749ab0 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: d449eb0d-fecc-48c5-a4bf-e68f3d353c4c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: 6c7f385d-ac39-46ca-9e91-7c7319e2458a kind: Event lastTimestamp: "2026-07-01T17:31:48Z" message: InferenceService [error-404-isvc-bbf0f] is Ready metadata: creationTimestamp: "2026-07-01T17:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:48Z" name: error-404-isvc-bbf0f.18be3ac251fcbbf7 namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: f65b5953-52d8-4371-a257-ede59d4440ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:10:16.686352Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:10:16Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be39957c76a850 namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: 22da5da7-1933-4134-9da4-40593784e4a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be39959660ec9b namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: ac76d629-0a9a-4a2c-8369-bb5ba69dae71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be399598fbde9c namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 50aa9826-339d-4525-b253-5385b0df5e31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be39959bf0275f namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d4135359-6cb6-43d2-a0c0-fc2de730c802 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be39959c6f6ec8 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 41c4b17a-b0ec-48e3-8f17-6bc02d532229 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be39959c79152b namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 76fde76d-c031-4151-8624-ad0ecf72dc99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be39959f8f1d71 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 822866e9-a043-4156-98ae-63fb29ad9bb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be3995a0220b91 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 166b8072-58dc-4ba0-b1ba-169bfab1b29c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:10:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:19:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:10:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:34Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be3995ec1e6e6c namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: ec535b39-29ea-44a7-9021-79eaa85deb39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be3a16adfb11d7 namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 2a183078-f050-48a9-a286-e3cdabbea642 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be3a16adfbd22b namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: 6509f3f1-7d1f-4255-b3ef-e6c7b78c31c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 318c4e47-af61-4745-be04-a6ca2637ed1e kind: Event lastTimestamp: "2026-07-01T17:19:34Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:34Z" name: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n.18be3a17607a6f42 namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 3ca3cbd3-0301-4248-923c-b3015e6276e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ce016-predictor-84dbff7cdb namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: af181037-72f7-4353-a81b-13f49621a7c9 kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: 'Created pod: error-404-isvc-ce016-predictor-84dbff7cdb-rdb8n' metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:16Z" name: error-404-isvc-ce016-predictor-84dbff7cdb.18be39957c126b09 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 6b2ee46a-39f4-49e1-95ff-20935cb10a19 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: a8c05662-a9cd-4348-9ff3-af83fd1521a3 kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Scaled up replica set error-404-isvc-ce016-predictor-84dbff7cdb from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:16Z" name: error-404-isvc-ce016-predictor.18be399579862b22 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: c11ae5fd-bfac-419a-8f3c-89baeb9fb0ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 67b15b3f-e5ca-44d7-aebe-dca9034758b6 kind: Event lastTimestamp: "2026-07-01T17:10:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:31Z" name: error-404-isvc-ce016-predictor.18be3998f7c022b2 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 18363a3f-8574-475c-a17c-ae2af6767098 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 67b15b3f-e5ca-44d7-aebe-dca9034758b6 kind: Event lastTimestamp: "2026-07-01T17:10:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:31Z" name: error-404-isvc-ce016-predictor.18be3998f8281182 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 4b451962-f4df-4132-a817-cbb5783b2992 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 67b15b3f-e5ca-44d7-aebe-dca9034758b6 kind: Event lastTimestamp: "2026-07-01T17:11:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:46Z" name: error-404-isvc-ce016-predictor.18be399c76d39c2f namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 988dff02-73d1-46e5-a50d-753fa397ee90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 67b15b3f-e5ca-44d7-aebe-dca9034758b6 kind: Event lastTimestamp: "2026-07-01T17:11:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:46Z" name: error-404-isvc-ce016-predictor.18be399c775ced8d namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: a6123fd2-ac33-4d4f-ad2e-dee96afba8ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce016 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 33c2f545-c625-46da-b16f-34cfe5452bf6 kind: Event lastTimestamp: "2026-07-01T17:11:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:04Z" name: error-404-isvc-ce016.18be3995799431fd namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 04fb430e-6810-4863-a3f9-156aad42ede3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 33c2f545-c625-46da-b16f-34cfe5452bf6 kind: Event lastTimestamp: "2026-07-01T17:11:04Z" message: InferenceService [error-404-isvc-ce016] is Ready metadata: creationTimestamp: "2026-07-01T17:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:04Z" name: error-404-isvc-ce016.18be39a0a7f460c1 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 4310e621-e4ad-4b67-b8d5-36582c2817cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:30:06.602502Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-db836-predictor-6996c59df8-gc6q8 to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaa88fe474f namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 96f208d4-96cd-4153-a20f-16aa5131133f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaa2d973b9 namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: ada7f531-22e8-4e44-b739-f3f7275c7bda reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaa4a010ee namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 4365f978-7657-4313-823c-e9fe0e366472 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaa8196917 namespace: kserve-ci-e2e-test resourceVersion: "25543" uid: 64bb1119-c2c7-48c5-978e-cb386eecb5ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaa8915d2a namespace: kserve-ci-e2e-test resourceVersion: "25545" uid: 86865460-d309-4862-8a93-1a07eff152df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaa89c4b36 namespace: kserve-ci-e2e-test resourceVersion: "25546" uid: 98f2b348-66b5-4120-9fd5-30522b9d83ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaab3e04f4 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: defe95c8-cba1-4e03-8447-3477d1beb790 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaaabb6f2a1 namespace: kserve-ci-e2e-test resourceVersion: "25548" uid: d405a532-8a97-4adf-b7e4-7c9cdd78fe82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:30:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:54Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3aaab936e92d namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: 0de353c8-71ff-4dff-aa8a-ee5ca0bbb072 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:47:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:26Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3b9c9301b7bd namespace: kserve-ci-e2e-test resourceVersion: "33363" uid: 3b578f88-2b7c-493e-a2f6-fc82e0a20f9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:47:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:26Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3b9c93024f9d namespace: kserve-ci-e2e-test resourceVersion: "33364" uid: f961d356-8bb4-47a0-b21f-1df0a7d84c51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db836-predictor-6996c59df8-gc6q8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 9a6ed0aa-189f-4c19-bfee-d052a47aa0e9 kind: Event lastTimestamp: "2026-07-01T17:47:30Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-01T17:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:30Z" name: error-404-isvc-db836-predictor-6996c59df8-gc6q8.18be3b9d91d3a3d2 namespace: kserve-ci-e2e-test resourceVersion: "33437" uid: 9df30be7-21bf-43e2-8721-8b662e220342 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-db836-predictor-6996c59df8 namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: 76030a52-7522-4b23-9133-87fd2be7392f kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'Created pod: error-404-isvc-db836-predictor-6996c59df8-gc6q8' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-db836-predictor-6996c59df8.18be3aaa88859dea namespace: kserve-ci-e2e-test resourceVersion: "25491" uid: 3485c6c4-ceff-407e-ad0c-1aaf92d2130f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: 5d1d293f-32da-4a39-87bf-666fe494eb62 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Scaled up replica set error-404-isvc-db836-predictor-6996c59df8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-db836-predictor.18be3aaa86e00425 namespace: kserve-ci-e2e-test resourceVersion: "25484" uid: c8faedac-b799-4906-a1c6-7e7438c428e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25482" uid: 961813b2-dffb-4dc4-9a1c-1adb7dd6ad92 kind: Event lastTimestamp: "2026-07-01T17:30:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:36Z" name: error-404-isvc-db836-predictor.18be3aae05704df7 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 9ee781e7-3607-43c8-8884-4ad1ea5359fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25482" uid: 961813b2-dffb-4dc4-9a1c-1adb7dd6ad92 kind: Event lastTimestamp: "2026-07-01T17:30:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:36Z" name: error-404-isvc-db836-predictor.18be3aae05df078a namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 453d6f6a-8186-4cf2-9ce7-1056c76c4f41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 961813b2-dffb-4dc4-9a1c-1adb7dd6ad92 kind: Event lastTimestamp: "2026-07-01T17:31:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:21Z" name: error-404-isvc-db836-predictor.18be3ab503348c65 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: 377fd95c-8547-461d-aeb5-8d5565054d76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 961813b2-dffb-4dc4-9a1c-1adb7dd6ad92 kind: Event lastTimestamp: "2026-07-01T17:31:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:21Z" name: error-404-isvc-db836-predictor.18be3ab50398c4a4 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: f9c9bb8d-8dd7-46da-8b32-9b7df12b1d59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: ea4cb38c-058c-4e72-b4a4-d49ffa953be5 kind: Event lastTimestamp: "2026-07-01T17:31:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:04Z" name: error-404-isvc-db836.18be3aaa86e227d9 namespace: kserve-ci-e2e-test resourceVersion: "26367" uid: a09ace03-1f32-418f-b2ba-5eb34ad67bef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: ea4cb38c-058c-4e72-b4a4-d49ffa953be5 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-db836": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db836": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-db836.18be3aaa8776a939 namespace: kserve-ci-e2e-test resourceVersion: "25487" uid: 187a0b3d-e0e3-4fb2-9fd3-916f11b19487 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: ea4cb38c-058c-4e72-b4a4-d49ffa953be5 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db836": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:06Z" name: error-404-isvc-db836.18be3aaa8776da5c namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: 6787173d-692f-480a-b815-a82380fba79f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: ea4cb38c-058c-4e72-b4a4-d49ffa953be5 kind: Event lastTimestamp: "2026-07-01T17:31:04Z" message: InferenceService [error-404-isvc-db836] is Ready metadata: creationTimestamp: "2026-07-01T17:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:04Z" name: error-404-isvc-db836.18be3ab806efeb90 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: e4db6b00-3e16-481c-93f6-273d0efbfa5f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:08:06.249100Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-875bcd459-pbj9r to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be39771dcc3c30 namespace: kserve-ci-e2e-test resourceVersion: "14084" uid: 1e5b07a4-5c29-4b59-8ad6-f1f6e8af2fa0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397737ddd808 namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: 44138bdb-89ac-4b07-8176-57d5be4f2461 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be39773a5160af namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 68bd3eac-db6b-47e1-bc56-02f6be2aa5e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:10Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" in 4.163s (4.163s including waiting). Image size: 301878744 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:10Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be3978327829b5 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: f97826bb-6f16-4025-9cd5-9f4badecb9cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:10Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397835b2016d namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: 8c18ed74-fd17-4729-948a-823bdc1c0041 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:10Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397836441ca3 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: b3ac802b-4374-4b4e-8279-584a7d316bc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:15Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T17:08:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:15Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be3979407bf7a6 namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: ea2d9b1d-79af-4d67-80ec-be7a073edc4c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 12.705s (12.705s including waiting). Image size: 1555871157 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397c35cf2490 namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 7c4c1377-0ef6-4f5b-94e8-8eec59d2d265 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397c38d3792d namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: c886db82-c4f9-46c6-89e5-040fbcbb9ded reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397c394c5458 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: aa455b9b-f986-41cf-ad55-24d4797ada9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397c3959baa4 namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: 9ec268e6-2ef3-4ee2-95a3-744e275a77fb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.187s (2.187s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:30Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397cbbb85b1a namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: 60b5b214-ef0b-49ed-9f88-7f4ab8d4ba92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:30Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397cbf267229 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 2b17d373-325f-4bee-a64f-1a7237fb39b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:08:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:30Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397cbf9c4d1a namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 1266e6d9-79b0-4831-8693-91ba23101743 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:09:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be397cffb591f4 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: cf3b0485-1c3d-4a84-ada1-cf3a61158d85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be39957735724b namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 5eae7019-9b14-4092-9749-add27483b245 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be399577383333 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: fad25f1c-3b24-422a-8ce9-8a3a7e7f5bf7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d1867322-cdf6-4580-b98f-156391fc5b4a kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r.18be3995ae064c77 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 4148c4fe-c518-4262-bd7f-a079084f8827 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-875bcd459 namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: a8278ee6-4e28-485c-9b57-5720b2bf3de3 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-875bcd459-pbj9r' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1-predictor-875bcd459.18be39771d208c57 namespace: kserve-ci-e2e-test resourceVersion: "14081" uid: 85e19f85-f8a6-4a1d-90ad-c66e6419410c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: a80c370c-7d87-48d7-9a70-17191f030a79 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-875bcd459 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1-predictor.18be39771493c7a1 namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: 7e3420f0-4f7f-4de1-9786-8bc82f2d8d23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14071" uid: 092534f4-595c-49e7-a9bd-2e90556c3d56 kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: isvc-sklearn-graph-1-predictor.18be397a9aa08148 namespace: kserve-ci-e2e-test resourceVersion: "14525" uid: 7f12f2d4-9ec6-49e1-82be-9005901d6a3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14071" uid: 092534f4-595c-49e7-a9bd-2e90556c3d56 kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: isvc-sklearn-graph-1-predictor.18be397a9b2425bb namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: c00db191-5449-42a6-9800-786b6a18ecc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 092534f4-595c-49e7-a9bd-2e90556c3d56 kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:06Z" name: isvc-sklearn-graph-1-predictor.18be39819958e9ba namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 4bdf42d4-4b2c-4ae1-aa26-b94575418a01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 092534f4-595c-49e7-a9bd-2e90556c3d56 kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:06Z" name: isvc-sklearn-graph-1-predictor.18be398199d665ad namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 8804e547-e6e8-438d-b5d4-0e9eb810399f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 1217b3f1-b63d-41f6-9b16-db0a5a390123 kind: Event lastTimestamp: "2026-07-01T17:09:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:44Z" name: isvc-sklearn-graph-1.18be39771acab477 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: fd1b7222-a208-434a-9ecc-8085f5c1c7df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 1217b3f1-b63d-41f6-9b16-db0a5a390123 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1.18be39771b281354 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: 76bb67b4-9741-4c4b-ac80-ada8aaa9f5a1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 1217b3f1-b63d-41f6-9b16-db0a5a390123 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-1.18be39771b2861ee namespace: kserve-ci-e2e-test resourceVersion: "14074" uid: 180f95ab-de3f-442d-829c-38dae69248af reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: 1217b3f1-b63d-41f6-9b16-db0a5a390123 kind: Event lastTimestamp: "2026-07-01T17:09:44Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T17:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:44Z" name: isvc-sklearn-graph-1.18be398de064de24 namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: bcd09b33-d12c-45d2-aa78-a0bbd94f74ed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:08:06.887581Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww to ip-10-0-140-168.ec2.internal metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397743daa028 namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: c19ce57b-88fe-4cb2-9206-2639ba5f8b6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:07Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be3977544fe179 namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: b247bf31-d293-473c-9637-ccb037316b82 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:08:07Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be3977818dc138 namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 0b56a916-8b3c-4051-b48e-61b22e9a0343 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:07Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be3977845f5b09 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: 649b7201-0256-49c7-88a0-eef8ad10e0c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:08Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397786f7c2c4 namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: 8a9f56cc-7351-4bc2-9d90-510f4e0fe714 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:08Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397787602631 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 0e49de94-d474-49e8-8743-d2d1e68470d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T17:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:12Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be39789c699bdc namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: b4f41062-76d9-4fdb-8ccc-5ca5af8a5c79 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:27Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 15.102s (15.102s including waiting). Image size: 1555871157 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:27Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397c209b644d namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 7d45baca-64a8-4143-a70c-7aae614ee19f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:27Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397c2409de62 namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: cab3df10-3130-4b2a-a460-0a4eed1d44d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:27Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397c2473ec7a namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 0b0cafeb-0a50-4291-bd2d-64c0fcde8e33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:27Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397c24801eda namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: b183df49-dc82-4517-b7af-f3e40d68c310 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 11.556s (11.556s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:39Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397ed5583a05 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: e112fede-e762-4815-8144-9c22f9ffb94d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:39Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397edc1a8d6e namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 930a90f2-7d8b-4a3e-8ecc-3cfd99a48664 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:08:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:39Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397edd047b77 namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 4639e237-7c84-4bbc-a527-dd6137c15b51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:09:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:35Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be397eee6ec543 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: bf685f9f-b5d4-48bc-8be0-ac978f377f2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be3995722dd2ca namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 9ce377c4-6635-485d-ac3e-77ee6cea4516 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be3995722e88ee namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 27edf4fa-73da-4b2f-bcb4-0005183e768d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4826e838-fb4b-49d9-9674-2e9f82278e7c kind: Event lastTimestamp: "2026-07-01T17:10:21Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-01T17:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:21Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww.18be3996ae70f64e namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: 7464313d-509e-45b9-845f-ae9807e8d015 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6c8478f9b5 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: e35c33e7-6fc7-4e5f-8de2-fd29adf45f3a kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6c8478f9b5-4vpww' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5.18be39774337b613 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 1cd46c46-2dac-46f7-bb55-538073c8297f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 3f207a71-a3ff-45a9-affd-532613dc47bf kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6c8478f9b5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-sklearn-graph-2-predictor.18be39773f2e2ea2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 21cd7d1f-4ddb-4c2a-9987-5dcc672a9c2e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 0b213ea6-c7a3-4c7b-b533-5037f03ba99a kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: isvc-sklearn-graph-2-predictor.18be397abdb9de0b namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 7dc7e2e2-253b-4784-9f0c-1ab1ec6626eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 0b213ea6-c7a3-4c7b-b533-5037f03ba99a kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: isvc-sklearn-graph-2-predictor.18be397abe2b0321 namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 37ee31a7-3557-43a3-b0a4-e2aaee4d1ed3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: 0b213ea6-c7a3-4c7b-b533-5037f03ba99a kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:06Z" name: isvc-sklearn-graph-2-predictor.18be3981bc454e20 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 395b4175-3c41-40dd-836c-1653397145e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: 0b213ea6-c7a3-4c7b-b533-5037f03ba99a kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:06Z" name: isvc-sklearn-graph-2-predictor.18be3981bcb2a685 namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: bdeb16f5-109b-4b1b-aca6-9d42af1dac13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: c950b31d-5698-4e56-92fc-8293c0e1b780 kind: Event lastTimestamp: "2026-07-01T17:09:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:45Z" name: isvc-sklearn-graph-2.18be39773f37193d namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 88335d22-a303-45cb-a756-602f864d97b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: c950b31d-5698-4e56-92fc-8293c0e1b780 kind: Event lastTimestamp: "2026-07-01T17:09:45Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T17:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:45Z" name: isvc-sklearn-graph-2.18be398e52da9cbd namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 048feb94-0f70-4d5a-80de-14322e8cbd1a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:08:06.571958Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14136" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-2p587 to ip-10-0-140-168.ec2.internal metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:08:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be3977310a9caf namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 7e248d76-ec2b-4561-894e-f99dfb05b99e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:08:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39774a7f3672 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 991e2c6d-140f-4020-ab4f-bdaab73c4295 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:067fec2d500e8a88ee10f3be03a35a3f15d715d25b8b6a966457243c4e7a45a0" already present on machine metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39774c1531e2 namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: ebfd83e1-b6ad-4681-908a-0c4f3d617f9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39774e9176ea namespace: kserve-ci-e2e-test resourceVersion: "14214" uid: 75aef539-c92f-4765-a49d-a48600221010 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T17:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39774ef5cbc9 namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: 3fc22e3f-a1e1-4055-85ed-0d331d508b93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:11Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T17:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be3978606666e7 namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 3dda314f-6ed3-4b5f-9dff-bddf450ae66b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:37Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 25.646s (25.646s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397e5910f986 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: a1107248-66f0-41c6-9a16-bdbef92c5ee7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397e60c872ab namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: d0b15eec-c7c2-4800-8984-a8f9806690c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397e61da64f2 namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 02534c52-a0fc-4b80-9122-d7cc62d6f600 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:37Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397e61e83d63 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 39258158-7b51-4002-b4b8-85378e2d5c07 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.937s (1.937s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397ed562fe04 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: c73910e5-4bd7-45e0-a9bd-c73237dc6273 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397edc021ad4 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 67b864e3-23ac-4a58-9ad0-fb97fc9ca321 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:08:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:08:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397edce7777f namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 9597b2e6-ca4c-48e1-8791-878ba746b61d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T17:08:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:08:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be397f2a67a36c namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 31e46f27-bb81-4bbb-8060-2f644a6770f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39956d1c86e9 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 373e913c-05da-40ef-bdb0-10e2f95d8921 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39956d1cd8fe namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: a9127da8-c517-4af9-bd9d-94d7c2c1b50c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2p587 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: c6adae20-bf8f-43a9-9596-0b15cc94ccbd kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-2p587.18be39958496cbd8 namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 7dabc1f9-4e6b-4f4e-ac04-f9d251163759 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-168.ec2.internal source: component: kubelet host: ip-10-0-140-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: ec3d60a5-d7f9-4453-a73c-d0a4872b3cce kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-2p587' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be3977301fa971 namespace: kserve-ci-e2e-test resourceVersion: "14137" uid: 7d515414-af59-4741-b12d-1d2627261e8d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14122" uid: 875cf45a-8f76-4723-a28c-f057fd520256 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: isvc-xgboost-graph-predictor.18be39772c91229f namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: a30dda66-0269-49dc-9232-349bdec9a88b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: e1448e2d-f39d-4fdb-9896-40418a11f86f kind: Event lastTimestamp: "2026-07-01T17:09:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:06Z" name: isvc-xgboost-graph-predictor.18be397aaa431994 namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: 32f7ed3a-c64f-4842-802d-c1c673807387 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: e1448e2d-f39d-4fdb-9896-40418a11f86f kind: Event lastTimestamp: "2026-07-01T17:09:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:06Z" name: isvc-xgboost-graph-predictor.18be397aaae62832 namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 7940ffab-f3b2-4259-bf28-4b1197f6635c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:09:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: e1448e2d-f39d-4fdb-9896-40418a11f86f kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:51Z" name: isvc-xgboost-graph-predictor.18be3988a699c927 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: ef146acf-b53c-4d8b-8075-44956027cb6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:09:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14371" uid: e1448e2d-f39d-4fdb-9896-40418a11f86f kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:51Z" name: isvc-xgboost-graph-predictor.18be3988a7066299 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 83bd2c71-c239-4c83-81da-9bbeb96e7526 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14107" uid: f2baeeda-3483-47d5-9dde-54b772d03e81 kind: Event lastTimestamp: "2026-07-01T17:09:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:37Z" name: isvc-xgboost-graph.18be39772c83ddb4 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 25f2144c-3998-4199-b1ea-8e68b3bb1802 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: f2baeeda-3483-47d5-9dde-54b772d03e81 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T17:09:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:36Z" name: isvc-xgboost-graph.18be398c3c8aacba namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 67b8b475-2c64-4aba-a637-d7b0b54e0113 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:10:06.126627Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-54fd6d96b6-gg5gt to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:10:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:10:06Z" name: model-chainer-54fd6d96b6-gg5gt.18be3993070e11b3 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: 9b950f89-73e0-4a06-b017-7ab2cef38b0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:06Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:10:06Z" name: model-chainer-54fd6d96b6-gg5gt.18be3993209f8cc0 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: 851bd28a-c037-454a-ba34-88e3edf043c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" metadata: creationTimestamp: "2026-07-01T17:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:06Z" name: model-chainer-54fd6d96b6-gg5gt.18be399321ef7bee namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: d3983d25-189e-469b-9833-0f0a0f03a9ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:09Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" in 2.593s (2.593s including waiting). Image size: 218070140 bytes.' metadata: creationTimestamp: "2026-07-01T17:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:09Z" name: model-chainer-54fd6d96b6-gg5gt.18be3993bc878e0f namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 54a41198-2bc7-42fc-afc3-e9f03baf9ca0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:09Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T17:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:09Z" name: model-chainer-54fd6d96b6-gg5gt.18be3993bfd75a60 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: af9dd161-9334-4a8c-9cc6-27d63cda2adb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:09Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T17:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:09Z" name: model-chainer-54fd6d96b6-gg5gt.18be3993c06708a2 namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 25520d54-6bf8-4f3a-9c6e-cd4ec97e7ab4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: model-chainer-54fd6d96b6-gg5gt.18be3995650f5fef namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 595de15b-567b-4972-a4d4-43a8f2b3ca95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-54fd6d96b6-gg5gt namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 781659ea-35ed-46c3-b4ce-ec0cd4625d61 kind: Event lastTimestamp: "2026-07-01T17:10:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:45Z" name: model-chainer-54fd6d96b6-gg5gt.18be39967b135205 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 32632a9a-fe88-4b0d-bc53-c8f5d7759465 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-54fd6d96b6 namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: df7558ea-9127-4f37-be85-84c88a9794e7 kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: 'Created pod: model-chainer-54fd6d96b6-gg5gt' metadata: creationTimestamp: "2026-07-01T17:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:06Z" name: model-chainer-54fd6d96b6.18be3993065844ae namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 9bf2014c-4127-4b2a-be13-151e5eef5fad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: cf376241-1ead-4e56-8a87-f44a680d2ecb kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: Scaled up replica set model-chainer-54fd6d96b6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:06Z" name: model-chainer.18be399305780ce5 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 475b3656-3eda-48d3-8564-674ccebd22bc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 762c2756-3eec-4c77-977b-fac8e64fd3a6 kind: Event lastTimestamp: "2026-07-01T17:10:06Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T17:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:06Z" name: model-chainer.18be39930afbf18c namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 4365b1c5-58f2-4b0a-bf88-e095793b8de6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:20:41.391687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-1f194-54bc64db78-46rl4 to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:20:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:20:41Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a26efcd96aa namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 0cc09a55-0c60-4155-af1c-73ab8f39ca63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:20:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-1f194-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:41Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a26f9639c5c namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 61b5e2b5-6d71-4775-bad6-83fd20e9618b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:20:42Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:20:42Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a272d83d16f namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 8094898d-cd58-4aea-873e-869d1f9c4aea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f194} kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:20:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:42Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a272ec660e1 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: bba4a305-3230-4ca7-ba85-96d7aef4b0ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f194} kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:20:42Z" message: 'Created container: sequence-graph-1f194' metadata: creationTimestamp: "2026-07-01T17:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:42Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a2732ba10f1 namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: cbc39492-7c31-4374-9a7b-39588b505dc3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f194} kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:20:42Z" message: Started container sequence-graph-1f194 metadata: creationTimestamp: "2026-07-01T17:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:42Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a27332f628e namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 299ca593-92f9-41fe-bbb1-3ef47f60c3dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f194} kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Stopping container sequence-graph-1f194 metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:51Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a294807e02b namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: 0de6cbd2-3ea3-4a98-a8f4-266fd393c924 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:20:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-1f194} kind: Pod name: sequence-graph-1f194-54bc64db78-46rl4 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 2e6ff716-5428-4626-bc17-81e7b5470ec9 kind: Event lastTimestamp: "2026-07-01T17:21:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:18Z" name: sequence-graph-1f194-54bc64db78-46rl4.18be3a29cdd777dc namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 147f7370-052f-4c7e-95f3-1b51a1f565da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-1f194-54bc64db78 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 8cd84efa-8d62-4ebe-89bd-9df19dd923af kind: Event lastTimestamp: "2026-07-01T17:20:41Z" message: 'Created pod: sequence-graph-1f194-54bc64db78-46rl4' metadata: creationTimestamp: "2026-07-01T17:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:41Z" name: sequence-graph-1f194-54bc64db78.18be3a26ef5813bc namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: a3089708-3902-4dcf-9e4c-996bbd5f9cd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-1f194 namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: fe48b527-7290-4c53-b642-373f7795dea6 kind: Event lastTimestamp: "2026-07-01T17:20:41Z" message: Scaled up replica set sequence-graph-1f194-54bc64db78 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:41Z" name: sequence-graph-1f194.18be3a26ee8d95b8 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 95ebb45a-ac13-4b3a-b586-d8c612a38107 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-1f194 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 82b54ec4-d02c-4a5e-bb8b-932b048e8aff kind: Event lastTimestamp: "2026-07-01T17:20:41Z" message: InferenceGraph [sequence-graph-1f194] is Ready metadata: creationTimestamp: "2026-07-01T17:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:41Z" name: sequence-graph-1f194.18be3a26f3c7f15b namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 0ecc2521-657e-464d-992d-ff9ab51f05b9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:21:51.691038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6460c-7c977b8767-wcnvq to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3a374df8a566 namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: e3f8a2f7-ddee-4f13-b230-73ef28c84fc4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:21:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-6460c-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3a375ac1b4d6 namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: 7f7d5133-fd74-4266-aa03-f9e5762164d8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:52Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:21:52Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:21:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:21:52Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3a378b8fd5d1 namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: 18b765c9-245d-4efa-9e6b-ea710e0583f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6460c} kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:21:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:21:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:52Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3a378cd030ef namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: d5ec0ee5-8ce8-4c9b-8072-01c7b03c5d87 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6460c} kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:21:52Z" message: 'Created container: sequence-graph-6460c' metadata: creationTimestamp: "2026-07-01T17:21:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:52Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3a378fc03f03 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 8ee45f11-f427-4bce-9ce1-17e91d43e7ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6460c} kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:21:52Z" message: Started container sequence-graph-6460c metadata: creationTimestamp: "2026-07-01T17:21:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:52Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3a3790315a07 namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: 99090d67-de6b-4327-9653-20f1c8de4294 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6460c} kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Stopping container sequence-graph-6460c metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:06Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3aaa74192a66 namespace: kserve-ci-e2e-test resourceVersion: "25410" uid: 17cfd11e-92b3-4e7b-a3e0-8194c01a6b06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6460c} kind: Pod name: sequence-graph-6460c-7c977b8767-wcnvq namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 3450c9cd-5280-457d-a637-cab3661fc1a6 kind: Event lastTimestamp: "2026-07-01T17:30:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:34Z" name: sequence-graph-6460c-7c977b8767-wcnvq.18be3aab18e0d490 namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 90f26cc0-63f0-4350-a11b-b412e72f04cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6460c-7c977b8767 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 478a0b6a-6bf8-4c88-a03c-f7140265f10b kind: Event lastTimestamp: "2026-07-01T17:21:51Z" message: 'Created pod: sequence-graph-6460c-7c977b8767-wcnvq' metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c-7c977b8767.18be3a374d449a1b namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 7082ea38-9f1a-42f1-8d43-b85d0f5725f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6460c namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: 55021697-cbbc-446c-aae2-9f874e9efebb kind: Event lastTimestamp: "2026-07-01T17:21:51Z" message: Scaled up replica set sequence-graph-6460c-7c977b8767 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c.18be3a374c790ead namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 17022943-7742-45c4-bf4d-319f7ee100a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6460c namespace: kserve-ci-e2e-test resourceVersion: "21689" uid: ce95fcb8-5a4f-48e8-bb9b-72672cee19d8 kind: Event lastTimestamp: "2026-07-01T17:21:51Z" message: InferenceGraph [sequence-graph-6460c] is Ready metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c.18be3a37526eb881 namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: b3aff339-1900-4de2-adbc-4bf6b987964f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6460c namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: ce95fcb8-5a4f-48e8-bb9b-72672cee19d8 kind: Event lastTimestamp: "2026-07-01T17:21:51Z" message: 'Failed to update status for InferenceGraph "sequence-graph-6460c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-6460c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c.18be3a37556cb777 namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: 1d3acea1-9bb9-4b12-8a16-a889890bd53e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6460c namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: ce95fcb8-5a4f-48e8-bb9b-72672cee19d8 kind: Event lastTimestamp: "2026-07-01T17:21:51Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-6460c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:51Z" name: sequence-graph-6460c.18be3a37556d11e1 namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: c5e84b78-c37b-4296-a2dc-a3bb24f9dbeb reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:22:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6460c namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 0c17affb-030e-48de-940a-e7dda601da2c kind: Event lastTimestamp: "2026-07-01T17:22:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:21Z" name: sequence-graph-6460c.18be3a3acacdf275 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 1fd8f1f2-6485-4564-9f00-e29a63fd3acd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:22:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6460c namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 0c17affb-030e-48de-940a-e7dda601da2c kind: Event lastTimestamp: "2026-07-01T17:22:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:21Z" name: sequence-graph-6460c.18be3a3acb3d75a4 namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: 00ce454d-ed88-4f78-80e6-34677e7ab92b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:11:16.454273Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ce016-7b6cbf87f8-94t72 to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be39a366e8d457 namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 874a866a-5716-41fe-b807-c6dc6a558d82 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be39a38080b8b8 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: 666a9f4b-da97-4faf-9157-f4701dbb0ba8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ce016} kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be39a381bd82ec namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 552697dc-a290-4786-8127-b457e9558063 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ce016} kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: 'Created container: sequence-graph-ce016' metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be39a384de4aba namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: 0f0ba80b-3281-4cde-afa6-85b210afe3f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ce016} kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: Started container sequence-graph-ce016 metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be39a38552d8ef namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: 0867e73c-da37-45c5-ba38-f13785ae5deb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ce016} kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Stopping container sequence-graph-ce016 metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be3a1698f6e059 namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: a9ed65d5-a971-4b68-871b-7588018b1da9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:19:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ce016} kind: Pod name: sequence-graph-ce016-7b6cbf87f8-94t72 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 4169310d-a7c3-4f9d-a766-d78b52814468 kind: Event lastTimestamp: "2026-07-01T17:19:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:19:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:58Z" name: sequence-graph-ce016-7b6cbf87f8-94t72.18be3a1712c5f5ae namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 5cd37752-6978-425b-91d3-4b179c8d17bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ce016-7b6cbf87f8 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: aa2d2e69-84e5-4c49-8bc7-295278526f3e kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: 'Created pod: sequence-graph-ce016-7b6cbf87f8-94t72' metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016-7b6cbf87f8.18be39a3666c8cc9 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 95076653-c7b8-4f0b-8c9c-269f4ee3c4aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: e2ab0a9a-3a28-4e77-97c1-6b9f2974d0ad kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: Scaled up replica set sequence-graph-ce016-7b6cbf87f8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016.18be39a3659e4569 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 717e1023-250d-4bbe-b0db-6279d9f4ff9b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: f919fa5b-9e92-42b5-8b91-ecc38cac3646 kind: Event lastTimestamp: "2026-07-01T17:11:16Z" message: InferenceGraph [sequence-graph-ce016] is Ready metadata: creationTimestamp: "2026-07-01T17:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:16Z" name: sequence-graph-ce016.18be39a36bb9bbfd namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: de5df72b-c0c0-48d1-a6a7-61e51fcb693b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 1c3864eb-eec5-4708-996b-ff6e0a80f23c kind: Event lastTimestamp: "2026-07-01T17:11:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:31Z" name: sequence-graph-ce016.18be39a6e3e6609f namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 2ee76754-5439-41bb-8ad2-a1f194a77eec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 1c3864eb-eec5-4708-996b-ff6e0a80f23c kind: Event lastTimestamp: "2026-07-01T17:11:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:31Z" name: sequence-graph-ce016.18be39a6e42f418a namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: fb525e52-015d-4045-b896-a8fb28894134 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 1c3864eb-eec5-4708-996b-ff6e0a80f23c kind: Event lastTimestamp: "2026-07-01T17:11:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:46Z" name: sequence-graph-ce016.18be39aa630345ba namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: d17ccb20-fd17-4314-9d96-7aa2c2918f73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 1c3864eb-eec5-4708-996b-ff6e0a80f23c kind: Event lastTimestamp: "2026-07-01T17:11:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:46Z" name: sequence-graph-ce016.18be39aa63752ed5 namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 475b8d6f-9104-4182-b8b8-96794da3092c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:30:50.706875Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4517b-7b778f989-h6fmq to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:30:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:30:50Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab4cdd18515 namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: d3ecba7a-eb6a-4d92-82fc-6ad99c94ed19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:50Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:30:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-4517b-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:30:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:50Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab4d7f6fe16 namespace: kserve-ci-e2e-test resourceVersion: "26078" uid: c82c4a44-6501-47aa-9ba7-7b63090f613f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:30:51Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:30:51Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab50af82032 namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: f4ba34c3-1702-456a-b3ff-f691d03a1495 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4517b} kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:30:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:51Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab50c4ac5ae namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 3334c597-e700-487e-b843-4ca477808237 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4517b} kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:30:51Z" message: 'Created container: splitter-graph-4517b' metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:51Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab50f42c535 namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: f3be680f-0da6-41f7-a2d2-70d1c34cee00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4517b} kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:30:51Z" message: Started container splitter-graph-4517b metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:51Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab50fb4cb71 namespace: kserve-ci-e2e-test resourceVersion: "26095" uid: 052ce03c-8d78-4907-8a23-e439b1f5e4a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4517b} kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:31:00Z" message: Stopping container splitter-graph-4517b metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:00Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab725e9095d namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: ae53a59a-3bc5-44b2-8717-70868f1e0914 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4517b} kind: Pod name: splitter-graph-4517b-7b778f989-h6fmq namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 6e1e5a4c-3139-4615-b8c1-6f1778839e95 kind: Event lastTimestamp: "2026-07-01T17:31:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:28Z" name: splitter-graph-4517b-7b778f989-h6fmq.18be3ab7c386ebe8 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: ce29719e-1a9b-42a1-bebe-2845b616b319 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4517b-7b778f989 namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: f347a0ce-920c-4196-8e4a-8fcf277a3d8b kind: Event lastTimestamp: "2026-07-01T17:30:50Z" message: 'Created pod: splitter-graph-4517b-7b778f989-h6fmq' metadata: creationTimestamp: "2026-07-01T17:30:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:50Z" name: splitter-graph-4517b-7b778f989.18be3ab4cd4c62fc namespace: kserve-ci-e2e-test resourceVersion: "26064" uid: 2eacc219-2796-4c8d-be7d-d38b30e0f7e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4517b namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 00b87acc-5925-4d59-807d-0f0dc9f3ae88 kind: Event lastTimestamp: "2026-07-01T17:30:50Z" message: Scaled up replica set splitter-graph-4517b-7b778f989 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:30:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:50Z" name: splitter-graph-4517b.18be3ab4cb841468 namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: 003c99dc-a15f-48a4-b121-3db9b7d9afab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4517b namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: e34318b3-982e-4c9d-98f0-9b2985bec377 kind: Event lastTimestamp: "2026-07-01T17:30:50Z" message: InferenceGraph [splitter-graph-4517b] is Ready metadata: creationTimestamp: "2026-07-01T17:30:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:50Z" name: splitter-graph-4517b.18be3ab4d2b15bc6 namespace: kserve-ci-e2e-test resourceVersion: "26076" uid: 1a1fc380-b975-4b53-ad1a-1db6664000b3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:32:01.004792Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-bbf0f-c98544745-zdsfs to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:32:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:32:01Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3ac52be6a7de namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 6a4c8303-fe71-4890-88e6-c538ce935b36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:01Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: "2026-07-01T17:32:01Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:32:01Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3ac5456936da namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: e15a42a2-ff1b-4a37-8f0d-12dcd11db2d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bbf0f} kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: "2026-07-01T17:32:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:32:01Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3ac546937916 namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: 1949c0b4-4696-48b8-8a98-a360aa69f4b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bbf0f} kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: "2026-07-01T17:32:01Z" message: 'Created container: splitter-graph-bbf0f' metadata: creationTimestamp: "2026-07-01T17:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:32:01Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3ac5497152e3 namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: 8a212246-ef09-41cb-beca-6836a628545a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bbf0f} kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: "2026-07-01T17:32:01Z" message: Started container splitter-graph-bbf0f metadata: creationTimestamp: "2026-07-01T17:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:32:01Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3ac549e07701 namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: b0ceb7fb-fb78-4bfc-b735-c75552af13ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bbf0f} kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: Stopping container splitter-graph-bbf0f metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3b38586717a9 namespace: kserve-ci-e2e-test resourceVersion: "30339" uid: 28f870f1-8c60-42c9-a259-83f0a9231af6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bbf0f} kind: Pod name: splitter-graph-bbf0f-c98544745-zdsfs namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: dc3420c9-9f51-498d-b2db-a47e62ce39d2 kind: Event lastTimestamp: "2026-07-01T17:40:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:42Z" name: splitter-graph-bbf0f-c98544745-zdsfs.18be3b38d462e1a3 namespace: kserve-ci-e2e-test resourceVersion: "30576" uid: 588ee482-8dc0-46a6-9d63-e2d6ab119496 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-bbf0f-c98544745 namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: e1a7ebc1-7b07-420b-be11-c38b5ecf5821 kind: Event lastTimestamp: "2026-07-01T17:32:00Z" message: 'Created pod: splitter-graph-bbf0f-c98544745-zdsfs' metadata: creationTimestamp: "2026-07-01T17:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:00Z" name: splitter-graph-bbf0f-c98544745.18be3ac52b2baff0 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: c744e02b-80c4-4ede-af69-0c6006e59029 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 2e2ef122-176e-4ea8-a78b-769eea39f779 kind: Event lastTimestamp: "2026-07-01T17:32:00Z" message: Scaled up replica set splitter-graph-bbf0f-c98544745 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:00Z" name: splitter-graph-bbf0f.18be3ac52a56b4f5 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 2e54e997-5e23-400c-8464-098c2864086d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:32:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: c497b782-5282-4d1d-aae8-870cdab449a9 kind: Event lastTimestamp: "2026-07-01T17:32:01Z" message: InferenceGraph [splitter-graph-bbf0f] is Ready metadata: creationTimestamp: "2026-07-01T17:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:32:01Z" name: splitter-graph-bbf0f.18be3ac52ffecb35 namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: 3b10813c-5995-4c2a-88dc-ff72efd3ef76 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:32:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 4addf8d3-0e92-458e-8e80-025f51a8ab48 kind: Event lastTimestamp: "2026-07-01T17:32:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:46Z" name: splitter-graph-bbf0f.18be3ac8a8c84bbb namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 8014779a-a47e-4fbc-81bb-6d09c397ddaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:32:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 4addf8d3-0e92-458e-8e80-025f51a8ab48 kind: Event lastTimestamp: "2026-07-01T17:32:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:46Z" name: splitter-graph-bbf0f.18be3ac8a936a0ad namespace: kserve-ci-e2e-test resourceVersion: "27230" uid: d991da6f-ceba-4c0e-b57f-8e2c37492602 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:08:06.398096Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14106" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:08:06Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397726ada7df namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: 9331eb40-0e80-42de-bc7e-108efe8f7254 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:08:06Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397741c72966 namespace: kserve-ci-e2e-test resourceVersion: "14187" uid: 4d7a6a09-8667-427f-87d5-361d2e411b5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:06Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be3977437e8454 namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 54570520-06ac-4dc3-a3f8-cbccff3f4144 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" in 18.529s (18.529s including waiting). Image size: 1335612528 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397b93e9c328 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 67720e2b-b5ba-434a-a3f7-6421a8ebe545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397b979d3d56 namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: e11999e8-657f-4bb8-81cb-fa3bc877b06f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397b9821ee7c namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 1841ce3f-b926-4344-825d-aa9993cba280 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:25Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T17:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:25Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397b982e8701 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: ab516e2d-b9cb-4039-91ec-30dbc83ed2f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.977s (2.977s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397c49adc67a namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 18f26099-9df9-48a9-b339-a3d9ddd4cf55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397c4d7f10ac namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: e4a729b9-8584-4e08-b2dd-1a98577f5f33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:08:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:08:28Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397c4dfa9abb namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 6d57eaba-f715-482b-b06f-24dd54b0d422 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:09:06Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:06Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be397cac2d7b76 namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: ad0a4519-fc3d-4be8-bb81-b28303a7c2d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be398f6b5f2223 namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: b64eb103-1b76-466f-b0ce-b2a1894ac8fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be398f6b605a38 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 3f828437-134e-4dac-8096-0786b2c5f6f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 440dce6a-9f80-46d4-a092-9a34c40759cf kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf.18be398f8859d61c namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: e146a043-aa5e-4743-90ad-32581cd5d43a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1b15a-predictor-84b9dd6865 namespace: kserve-ci-e2e-test resourceVersion: "14097" uid: 1742bc65-046e-4601-8e47-cf06e59a5910 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: 'Created pod: success-200-isvc-1b15a-predictor-84b9dd6865-v2nnf' metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: success-200-isvc-1b15a-predictor-84b9dd6865.18be397726056ea7 namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: d45f1c48-efe1-48ef-9225-af2115fb222f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 284604ff-fcb1-4348-a658-39b2a55c2b46 kind: Event lastTimestamp: "2026-07-01T17:08:06Z" message: Scaled up replica set success-200-isvc-1b15a-predictor-84b9dd6865 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:06Z" name: success-200-isvc-1b15a-predictor.18be397722f8137a namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: a0ce212a-7544-40ec-9d32-c911bd54e1f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 1f9fc823-d834-47ed-9fd6-8aa431566dcd kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: success-200-isvc-1b15a-predictor.18be397aa1792098 namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 6d1ba136-d543-4e9b-9a72-2ca1cb6bed02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:08:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 1f9fc823-d834-47ed-9fd6-8aa431566dcd kind: Event lastTimestamp: "2026-07-01T17:08:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:08:36Z" name: success-200-isvc-1b15a-predictor.18be397aa20f70e6 namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: b96681ea-6fd8-4847-ad77-c6bffec08032 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: 1f9fc823-d834-47ed-9fd6-8aa431566dcd kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:36Z" name: success-200-isvc-1b15a-predictor.18be3981a0f8d83c namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: fa7ef243-0017-4ba6-bf05-407f7f4f572b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:08:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b15a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: 1f9fc823-d834-47ed-9fd6-8aa431566dcd kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:36Z" name: success-200-isvc-1b15a-predictor.18be3981a1bc7fc1 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: fb735633-e41b-4d48-ae3a-c54dd14a5e53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:08:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1b15a namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: 30e49f9c-f54b-4679-9200-f4084c4e0d24 kind: Event lastTimestamp: "2026-07-01T17:09:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:16Z" name: success-200-isvc-1b15a.18be3977237b75c9 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: bfd85873-741a-48c7-b4f9-1e2336730c70 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1b15a namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 30e49f9c-f54b-4679-9200-f4084c4e0d24 kind: Event lastTimestamp: "2026-07-01T17:09:16Z" message: InferenceService [success-200-isvc-1b15a] is Ready metadata: creationTimestamp: "2026-07-01T17:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:16Z" name: success-200-isvc-1b15a.18be398767faa01e namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 197e5200-cc75-4698-82c3-a8924085f0b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:19:31.380926Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1f194-predictor-6dd685749f-bq5w2 to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:19:31Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16a2d64f72 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 1049bdf9-4ee7-4f99-aa2c-8f3c46709e5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1f194-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16ab993a34 namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: 183637be-5e9c-445e-960b-cfebfe3c2dca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16e0ce2200 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 059005ab-989b-45ca-9039-2ad811839041 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16e243cdfb namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 2515cd38-a98a-48c8-b55e-aa62536de208 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16e675aa36 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: b516c857-8e70-4e89-9d41-2355c199e57f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16e7120f5c namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: cec98864-2019-4c1d-8a6d-ca0c316ee563 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16e71de7dd namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: c737201e-b3a3-4ee2-9bde-669c0da4a10e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16ea0f1a65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 3cde1804-7ef2-4b41-ab33-fc3b77c18fd8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:19:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:32Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16eaa43dcc namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: f990e775-d94b-4f2b-ac9d-7cb8370d1d55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:20:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:18Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a16f723921f namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: e95b435e-167b-4854-91fa-42153e8517d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:51Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a29515969da namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 4663691e-d59d-4a0a-97c1-9c7545b1dc94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:51Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a29515aa4d7 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 7aba55eb-09f1-4171-abe1-4819d24742ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: acb61295-6868-4bc3-9a75-77fae13527fd kind: Event lastTimestamp: "2026-07-01T17:20:53Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:53Z" name: success-200-isvc-1f194-predictor-6dd685749f-bq5w2.18be3a29d3541601 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 973aab7b-c1d5-423a-9fb2-ea09e04c059e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1f194-predictor-6dd685749f namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: 1d2e9d62-13f5-4188-9322-fb1ab3e7df92 kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: 'Created pod: success-200-isvc-1f194-predictor-6dd685749f-bq5w2' metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:31Z" name: success-200-isvc-1f194-predictor-6dd685749f.18be3a16a21f6ffe namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 06818ae9-7d9e-4f0e-b145-98675e2379cc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 141462b8-f8bb-42eb-b7d8-22b1599ede0a kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Scaled up replica set success-200-isvc-1f194-predictor-6dd685749f from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:31Z" name: success-200-isvc-1f194-predictor.18be3a16a00368f4 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 7576183b-eff2-41be-8c0b-480ef6619cf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:19:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 000abc28-edca-4e89-9e35-27572c5cf3a1 kind: Event lastTimestamp: "2026-07-01T17:20:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:16Z" name: success-200-isvc-1f194-predictor.18be3a1a1e91ccc8 namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: ee0abf4c-a6f8-42f3-be72-15a5749eb1f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:19:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 000abc28-edca-4e89-9e35-27572c5cf3a1 kind: Event lastTimestamp: "2026-07-01T17:20:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:16Z" name: success-200-isvc-1f194-predictor.18be3a1a1f087df3 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: f7adc59f-e76f-4ecf-a2ed-60a1427944b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 000abc28-edca-4e89-9e35-27572c5cf3a1 kind: Event lastTimestamp: "2026-07-01T17:20:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:46Z" name: success-200-isvc-1f194-predictor.18be3a1d9ddcf600 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: 13bba2fd-6599-4222-8eb0-7ef3de1387aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f194-predictor namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 000abc28-edca-4e89-9e35-27572c5cf3a1 kind: Event lastTimestamp: "2026-07-01T17:20:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:46Z" name: success-200-isvc-1f194-predictor.18be3a1d9e4da3b5 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 86dca475-940c-4db5-86dd-180a844a0c88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f194 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: 6652a617-3fd7-442b-9d66-c79d5a5e4498 kind: Event lastTimestamp: "2026-07-01T17:20:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:28Z" name: success-200-isvc-1f194.18be3a16a0148c28 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 3a2b7ec3-ba8f-43af-a0e3-3b0660bf44e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f194 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 6652a617-3fd7-442b-9d66-c79d5a5e4498 kind: Event lastTimestamp: "2026-07-01T17:20:28Z" message: InferenceService [success-200-isvc-1f194] is Ready metadata: creationTimestamp: "2026-07-01T17:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:28Z" name: success-200-isvc-1f194.18be3a24070fb2c4 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: e1438e41-47ad-4ab5-896b-eacc09e3d6d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:09:50.585528Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3c336-predictor-547bb6bc8f-96z6b to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398f68bbc3eb namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: f7ff7c8a-4a95-40da-9c9a-b7ae13db0fca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3c336-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398f737af0dc namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 6ca8aad0-a009-4171-a2b4-af951099077d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fa6b30b71 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: d97bb354-c2ab-415b-9be0-b0c2adf2fb73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fa86684ea namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: 55496775-864b-45f4-bbac-2cafc88237ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fab563689 namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: 0dae26b3-0c63-48fd-945d-add2cafa03c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fabd033bd namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: c52b115c-89dd-4fe2-80f8-769c0b5bac76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fabddfe62 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: 62c99e22-c345-47f0-97b2-cfdbf185a11a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398faf474ef5 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 7f4c24d2-b1b5-40f7-989f-44e75200baa3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:09:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:51Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fafdea808 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 63de28aa-13ec-4bdb-879a-bacfe977648c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T17:09:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:19:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:09:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:18Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be398fd72417a6 namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: c7eef760-6c1c-48bd-8b92-1e69ff3a0267 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be3a12f201af62 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 86ec35e2-2936-46e2-a766-0e4e777d5a9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be3a12f201ee57 namespace: kserve-ci-e2e-test resourceVersion: "19709" uid: 970768e8-cf7a-45ae-8887-893faa5c214f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dc760277-8744-4acb-89dc-4cbc32f83742 kind: Event lastTimestamp: "2026-07-01T17:19:18Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:18Z" name: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b.18be3a139f8fc798 namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 2293f6e9-778d-4bc6-a719-3010a090bea0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3c336-predictor-547bb6bc8f namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 47484ce5-b8db-44ad-ae08-96816551410c kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'Created pod: success-200-isvc-3c336-predictor-547bb6bc8f-96z6b' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-3c336-predictor-547bb6bc8f.18be398f6828565a namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 78f65805-34df-4709-9edd-1da37db9002b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 4fac6416-bfba-4b73-827f-6d171bf96dd0 kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Scaled up replica set success-200-isvc-3c336-predictor-547bb6bc8f from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-3c336-predictor.18be398f64ccf32a namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 9927f3a4-e1f0-4b96-991f-350ce1c2e340 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 77ec39b8-d608-40e9-b833-0450edcd1463 kind: Event lastTimestamp: "2026-07-01T17:10:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:20Z" name: success-200-isvc-3c336-predictor.18be3992e3511703 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 1eab6b2c-1c6a-49c5-afc3-d8198003eef3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 77ec39b8-d608-40e9-b833-0450edcd1463 kind: Event lastTimestamp: "2026-07-01T17:10:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:20Z" name: success-200-isvc-3c336-predictor.18be3992e3b35740 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: f9bcc4b3-834c-4596-821e-cd53dfdcab3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 77ec39b8-d608-40e9-b833-0450edcd1463 kind: Event lastTimestamp: "2026-07-01T17:11:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:05Z" name: success-200-isvc-3c336-predictor.18be3999e11a86a5 namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 41c074b0-9333-432e-bd12-ed26612f6c05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:10:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c336-predictor namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 77ec39b8-d608-40e9-b833-0450edcd1463 kind: Event lastTimestamp: "2026-07-01T17:11:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:05Z" name: success-200-isvc-3c336-predictor.18be3999e182c5a5 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 0f0c99d7-4892-4717-8c64-ebb763f77c2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 299a2e51-2771-4ac4-aa9a-093f2a6a26bb kind: Event lastTimestamp: "2026-07-01T17:10:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:48Z" name: success-200-isvc-3c336.18be398f64d2c323 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 2a3e2718-6a5b-4c79-b4d9-8cb69785d92e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 299a2e51-2771-4ac4-aa9a-093f2a6a26bb kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-3c336": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3c336": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-3c336.18be398f654360b7 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 168ea2aa-7e7b-4e35-b36b-7bf500cfaf2e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 299a2e51-2771-4ac4-aa9a-093f2a6a26bb kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3c336": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:50Z" name: success-200-isvc-3c336.18be398f6543ab02 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 162b78e7-afe2-43f3-9568-38da715ff5cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c336 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 299a2e51-2771-4ac4-aa9a-093f2a6a26bb kind: Event lastTimestamp: "2026-07-01T17:10:48Z" message: InferenceService [success-200-isvc-3c336] is Ready metadata: creationTimestamp: "2026-07-01T17:10:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:10:48Z" name: success-200-isvc-3c336.18be399ce6e639d9 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: adf5c35e-b1ca-4d73-8e9f-1653684c9e35 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:29:40.638346Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24952" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa47d68a663 namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 38f14f36-309e-40c2-b8f1-b57f352fd366 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24972" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa4973e421c namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: edb14714-996f-4159-89b4-b8b278de20a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa499113680 namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: 507d29bb-c1c1-405e-88e5-f9b7f13a0b19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa49c0f3b5b namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: d0213f0f-0637-44d8-97c1-a4159e3c769f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa49c83df37 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 6c1573da-9fdf-4e89-8b06-c6c29f9f1f0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa49c8e4c93 namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: 6bdccda4-9fe9-4c59-ab89-bc556bb2aaa6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa49f8ee98f namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: 8d10b7e3-09ce-4d3b-a10f-92a7881d4fca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:29:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:41Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa4a006630a namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: a1a74a48-9b01-454b-b5a8-a798e009f106 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:30:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:27Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3aa4a3f84bbb namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: 8edd70e0-7bbd-4dd2-b0e8-e561d74aef1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:31:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:00Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3ab72f549570 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 83f4441b-4ce8-47c4-9ffe-47bbd2a37fc5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:31:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:00Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3ab72f5533e8 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 96f7f93d-1581-42da-84e9-408ad667b20c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 834629ee-e276-4d11-934b-2b6a7179ee66 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm.18be3ab78075a8db namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: e6cb7296-d0a0-4bce-a4fb-dd1ab0f6ede2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4517b-predictor-64b9b9f5bc namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: b8e83c9a-3614-469d-a2c1-f322261811ac kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'Created pod: success-200-isvc-4517b-predictor-64b9b9f5bc-9pcrm' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-4517b-predictor-64b9b9f5bc.18be3aa47cc80c7e namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: b042e0cd-0136-4ea8-87f1-7bc81c1edbea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24932" uid: 95957ace-6705-438a-9e74-48fb35223de4 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Scaled up replica set success-200-isvc-4517b-predictor-64b9b9f5bc from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-4517b-predictor.18be3aa47922974d namespace: kserve-ci-e2e-test resourceVersion: "24941" uid: f499ed36-1c62-4fbf-b901-e351402186b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24937" uid: 23e90ea0-0f64-4f69-b66e-66a034692f21 kind: Event lastTimestamp: "2026-07-01T17:30:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:10Z" name: success-200-isvc-4517b-predictor.18be3aa7f7b40f73 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: ff2a9fc5-3dc4-4775-a487-8f510321e61d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:29:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "24937" uid: 23e90ea0-0f64-4f69-b66e-66a034692f21 kind: Event lastTimestamp: "2026-07-01T17:30:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:10Z" name: success-200-isvc-4517b-predictor.18be3aa7f82529ca namespace: kserve-ci-e2e-test resourceVersion: "25682" uid: adf20113-7bbd-4087-b71f-67fc1c2716e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 23e90ea0-0f64-4f69-b66e-66a034692f21 kind: Event lastTimestamp: "2026-07-01T17:30:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:55Z" name: success-200-isvc-4517b-predictor.18be3aaef5a7cf41 namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: a010b9ae-8cf4-4d9b-93e5-416f9ca9a263 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4517b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 23e90ea0-0f64-4f69-b66e-66a034692f21 kind: Event lastTimestamp: "2026-07-01T17:30:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:55Z" name: success-200-isvc-4517b-predictor.18be3aaef618bd4d namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: f8d2447f-78f3-47df-b9d3-62dc6d448f71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 2a0608a9-403d-4a1b-aba9-c863b7921083 kind: Event lastTimestamp: "2026-07-01T17:30:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:37Z" name: success-200-isvc-4517b.18be3aa4792fcbd4 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: c0245a98-e9b3-4301-99ed-a6a018f63b1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 2a0608a9-403d-4a1b-aba9-c863b7921083 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-4517b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4517b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-4517b.18be3aa479a6b923 namespace: kserve-ci-e2e-test resourceVersion: "24946" uid: 33c46d3f-4e04-4249-9b8b-e2ae4c2b2b38 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 2a0608a9-403d-4a1b-aba9-c863b7921083 kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4517b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-4517b.18be3aa479a713c1 namespace: kserve-ci-e2e-test resourceVersion: "24948" uid: c0dc8df8-3e66-449f-bf93-545235dc11e7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4517b namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: 2a0608a9-403d-4a1b-aba9-c863b7921083 kind: Event lastTimestamp: "2026-07-01T17:30:37Z" message: InferenceService [success-200-isvc-4517b] is Ready metadata: creationTimestamp: "2026-07-01T17:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:37Z" name: success-200-isvc-4517b.18be3ab1b4207eeb namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: b2bfdf3a-ebd2-4f3b-ae88-d26daecfd932 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:19:15.522599Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-52e43-predictor-674cd6cc9-prtkw to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a12f19b6568 namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 87a4059d-739a-403f-9586-f8143bee2bc3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-52e43-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a12fb9aa64f namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 390ab50e-2ca7-4cc7-8f83-e94e26500aea reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a132fd0999b namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: 4a68027d-bda5-4005-8a09-486c6235d9f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a13313cc345 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 9399d5ad-523c-4738-8c29-2ce0fa7b82bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a13346e4c30 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 81a8d2cf-d198-4e43-8ea7-99caec62f34e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a1334ff02ea namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: f955e949-f134-4a14-8843-3c4b85f8ed63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a1335092bc3 namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 76b2dcc3-c538-4238-9200-0bc941498ee5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a133909ecf4 namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: 748d8511-712a-4c2f-bedf-888acdcde9fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:16Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a13398bf9e4 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 9418207c-10ec-46f0-99ea-75c0e5386c9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:19:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:54Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a13746ebdfb namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: eae6b129-3de9-41b0-abf5-814f569a6624 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:25Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a234c730f39 namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 96c214e2-2106-4fbc-961d-624e04e3d34b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 430b61ee-095e-470f-a487-7d790277b37d kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:25Z" name: success-200-isvc-52e43-predictor-674cd6cc9-prtkw.18be3a234c738a98 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 39e084d1-6cce-42b9-9bf8-6fee479d8f58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-52e43-predictor-674cd6cc9 namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: a7e0311e-fd0b-47d3-bbca-7e914ee2ea64 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'Created pod: success-200-isvc-52e43-predictor-674cd6cc9-prtkw' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-52e43-predictor-674cd6cc9.18be3a12f13a4cad namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: aab7b802-2f1d-4f99-8afe-f1e27102727e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 40036eb5-db01-4aab-b0de-d196370afe69 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Scaled up replica set success-200-isvc-52e43-predictor-674cd6cc9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-52e43-predictor.18be3a12ed4739a3 namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: a5f16647-d998-4d3a-9ec5-2f6d0b0054b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: d18f070f-e255-4ac0-b8ef-e63b227921f9 kind: Event lastTimestamp: "2026-07-01T17:19:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:30Z" name: success-200-isvc-52e43-predictor.18be3a166bbe414e namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 36e8ce89-671d-4974-9975-525b195aa42f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: d18f070f-e255-4ac0-b8ef-e63b227921f9 kind: Event lastTimestamp: "2026-07-01T17:19:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:19:30Z" name: success-200-isvc-52e43-predictor.18be3a166c2e5d26 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: 35e0ce9f-f85a-480a-8712-0d6da2fe7086 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:19:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: d18f070f-e255-4ac0-b8ef-e63b227921f9 kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:15Z" name: success-200-isvc-52e43-predictor.18be3a19ebe88499 namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 83f264ec-335f-4028-a737-791d3c100e27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:19:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52e43-predictor namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: d18f070f-e255-4ac0-b8ef-e63b227921f9 kind: Event lastTimestamp: "2026-07-01T17:20:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:15Z" name: success-200-isvc-52e43-predictor.18be3a19ec655f81 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: cf105ca3-1843-4bf8-a4b2-4b559a63fe3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: 33708838-a047-4830-8f9c-c94ed47e708f kind: Event lastTimestamp: "2026-07-01T17:20:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:04Z" name: success-200-isvc-52e43.18be3a12ed555ae3 namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 207b9728-f97b-485a-8868-2f75c6d2cb6e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: 33708838-a047-4830-8f9c-c94ed47e708f kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-52e43": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-52e43": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-52e43.18be3a12edeffeb6 namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: fb6727cd-8f86-4c54-a500-53f9c4e62f34 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: 33708838-a047-4830-8f9c-c94ed47e708f kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-52e43": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:19:15Z" name: success-200-isvc-52e43.18be3a12edf03365 namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: 48bad3bc-5992-416e-ae6e-ba48567620fb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52e43 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 33708838-a047-4830-8f9c-c94ed47e708f kind: Event lastTimestamp: "2026-07-01T17:20:04Z" message: InferenceService [success-200-isvc-52e43] is Ready metadata: creationTimestamp: "2026-07-01T17:20:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:20:04Z" name: success-200-isvc-52e43.18be3a1e6bd59c84 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 6dedb5cc-522d-43d5-871e-77080d67c1d7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:20:51.624479Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6460c-predictor-5774675f9-x8h72 to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:20:51Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a2951b9b763 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 0885d69f-3626-4102-8f3e-dc44262c9214 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6460c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:51Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a295e4bb0b3 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 3b51888b-6cd6-4202-8760-12db04510d9d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a298f8b009b namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: c52a2fce-a3d6-4c33-9757-9720d1cf45c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a2991363f54 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: e3dd3d12-e3d4-447e-9fbb-44e1cb651019 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a29949a2a77 namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 26889e62-1547-4ad4-9003-8e450f8e3041 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a2995112e6d namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 230965d7-ece4-4845-bbf0-255beaefcc76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a29951bf892 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: afd89c0e-ac41-4e5f-8dde-bf3eff5bd774 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a299827b45b namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 63dc3200-acd2-4e1e-aad5-4b14fc03c076 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:20:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:52Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a2999330c46 namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: cf946767-e92d-4031-b36c-f4e81d718089 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:21:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:21:30Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3a29e9be9c48 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: b5207181-41fc-4581-835e-06447a100798 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3aaa7f95e3a5 namespace: kserve-ci-e2e-test resourceVersion: "25460" uid: 6e96aaf3-9284-4a0d-874c-0335428df9f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6460c-predictor-5774675f9-x8h72 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 5bb651ab-06a8-4dbc-812f-1eb6ac74f174 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-6460c-predictor-5774675f9-x8h72.18be3aaa7f9675b0 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: a829c6bd-2f6b-4d85-b362-dbfc7250fe2b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6460c-predictor-5774675f9 namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 88006730-92ff-495b-be9c-80b57a5ffa80 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: 'Created pod: success-200-isvc-6460c-predictor-5774675f9-x8h72' metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:51Z" name: success-200-isvc-6460c-predictor-5774675f9.18be3a29514dfc52 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 75c003c3-ac0c-4f16-8f98-c987ae78733d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 118db731-cbce-4fc6-b3f2-6af2d505c828 kind: Event lastTimestamp: "2026-07-01T17:20:51Z" message: Scaled up replica set success-200-isvc-6460c-predictor-5774675f9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:51Z" name: success-200-isvc-6460c-predictor.18be3a294ec8ff6d namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 57cfad75-73d3-4fd2-bba1-5b9b27e5b39a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 25e6fd70-786e-4c1b-bc81-69205be34d65 kind: Event lastTimestamp: "2026-07-01T17:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:21Z" name: success-200-isvc-6460c-predictor.18be3a2ccd9110c7 namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: ceff70af-380c-44b4-8e64-addf28c1381f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 25e6fd70-786e-4c1b-bc81-69205be34d65 kind: Event lastTimestamp: "2026-07-01T17:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:21Z" name: success-200-isvc-6460c-predictor.18be3a2cce01b863 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 1a01a340-2907-4f9f-8150-9341af99ce59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 25e6fd70-786e-4c1b-bc81-69205be34d65 kind: Event lastTimestamp: "2026-07-01T17:22:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:06Z" name: success-200-isvc-6460c-predictor.18be3a33cc05803b namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: 7662c0f9-dafb-4ec0-bac8-2c30a9b5084a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6460c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 25e6fd70-786e-4c1b-bc81-69205be34d65 kind: Event lastTimestamp: "2026-07-01T17:22:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:22:06Z" name: success-200-isvc-6460c-predictor.18be3a33cc6e660d namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 5016533e-7023-4e5a-b3b7-ec48e6293933 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6460c namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 4276f178-39fa-4f4d-ad2a-e62ff59c1bcd kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:40Z" name: success-200-isvc-6460c.18be3a294ec96d50 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 63008cc8-3fb1-47ea-b9a4-dbb04ee2134b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6460c namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: 4276f178-39fa-4f4d-ad2a-e62ff59c1bcd kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: InferenceService [success-200-isvc-6460c] is Ready metadata: creationTimestamp: "2026-07-01T17:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:40Z" name: success-200-isvc-6460c.18be3a34a65a23f8 namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 57a3a07f-eda2-4923-8a68-d66250d5dde2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:20:25.789236Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:20:25Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a234dd3334b namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: b55a0804-9c52-4022-a75d-f2686ef59342 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a2368ee9704 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 1759e63b-fb01-4e89-9f77-6baa52aa17b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a236a5bfdf4 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 280fd5bd-d66d-4631-8dee-f3c38154b91f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a236e6f254e namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 79ef329c-1f3d-463e-bd14-c411c864344c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a236eeda0bc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: a7e08cb8-e31f-4bb1-a606-7f2c9892092a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a236efa175d namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: 535b0ae5-fd7c-4480-823f-718350aeabf8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a2371eb0dec namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 9b1579a9-97bf-420a-b33a-539a9dad0742 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:20:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:20:26Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a2372691139 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 44eab449-f4f4-4d60-8316-4b4289bbca29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:20:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:29:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:20:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:43Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3a2398ae2097 namespace: kserve-ci-e2e-test resourceVersion: "25055" uid: bcba4020-a65b-44ae-a403-797c7efc76dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3aa479972fad namespace: kserve-ci-e2e-test resourceVersion: "24945" uid: 18ee4bc9-1a35-45be-bbaa-3c56907347f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:29:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:40Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3aa47997d80e namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: 2d9e9205-f5e4-4419-9927-4063c389037c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:29:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 19df0b11-647e-4826-99c7-a3016979c15b kind: Event lastTimestamp: "2026-07-01T17:29:43Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:29:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:29:43Z" name: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk.18be3aa50d320523 namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: c6614600-904b-4f3b-9e1d-8ef209d09706 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bbcac-predictor-7874fb45b7 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 8ac9c5db-2436-4b05-bb54-91b960822a40 kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: 'Created pod: success-200-isvc-bbcac-predictor-7874fb45b7-bmwwk' metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:25Z" name: success-200-isvc-bbcac-predictor-7874fb45b7.18be3a234d48d92f namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: e8d854ab-0b47-4ad9-9fa9-f1cb5f372a5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 1798f8a3-1880-400c-94fa-582f7f1796ed kind: Event lastTimestamp: "2026-07-01T17:20:25Z" message: Scaled up replica set success-200-isvc-bbcac-predictor-7874fb45b7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:25Z" name: success-200-isvc-bbcac-predictor.18be3a234b115d86 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 51c03ae6-3a73-4480-9298-50b42993179e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:20:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 04b8c828-aa78-4736-a37b-a0cccc72c2b4 kind: Event lastTimestamp: "2026-07-01T17:20:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:55Z" name: success-200-isvc-bbcac-predictor.18be3a26c9544aa1 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: a463324b-a27b-4e74-b66d-4b11c1fe3cd7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:20:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 04b8c828-aa78-4736-a37b-a0cccc72c2b4 kind: Event lastTimestamp: "2026-07-01T17:20:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:20:55Z" name: success-200-isvc-bbcac-predictor.18be3a26c9c2b9d5 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 270346a4-0a5f-4aac-b758-6823bebbb24e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 04b8c828-aa78-4736-a37b-a0cccc72c2b4 kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:40Z" name: success-200-isvc-bbcac-predictor.18be3a2dc7558fbb namespace: kserve-ci-e2e-test resourceVersion: "21588" uid: 0a3b173c-41c5-4658-984d-f505b05ef7eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:21:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbcac-predictor namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 04b8c828-aa78-4736-a37b-a0cccc72c2b4 kind: Event lastTimestamp: "2026-07-01T17:21:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:21:40Z" name: success-200-isvc-bbcac-predictor.18be3a2dc7c2dee9 namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: 92f0d077-b378-49c5-a64c-600715423ace reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T17:20:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bbcac namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 53ee08a6-549b-4b5f-a3bb-e7404008819e kind: Event lastTimestamp: "2026-07-01T17:21:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:13Z" name: success-200-isvc-bbcac.18be3a234b069aad namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 32cd7999-9ade-4189-a675-2787aae5d358 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:21:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bbcac namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 53ee08a6-549b-4b5f-a3bb-e7404008819e kind: Event lastTimestamp: "2026-07-01T17:21:13Z" message: InferenceService [success-200-isvc-bbcac] is Ready metadata: creationTimestamp: "2026-07-01T17:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:21:13Z" name: success-200-isvc-bbcac.18be3a2e54f7ca03 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 99410837-bd2b-494b-929a-28c4beecb5ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:31:00.933627Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:31:00Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab72f616d30 namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: fe6eddd2-a155-4378-91d8-9e3606935c64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-bbf0f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:01Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab73b7a9e30 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: cbff4f8d-8f19-449f-a1e2-f6150bfa6ee8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:01Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:31:01Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab76d919b72 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: 7dab6789-5a7b-4d42-a001-f637920a285a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab76f05a308 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: 4affed25-4bb8-4ed0-a345-e7eff5cb91c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab772d14889 namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: bb6ef8c3-80c7-4d63-b50f-5ed4945e87b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab7734d2164 namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: 72db703f-93a8-4d12-a473-7660a5567041 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab7735d555c namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: 78c0ad6c-f7e7-486e-8e37-7ac625f13907 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab7763f5fc1 namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 3cf485a8-2a79-478b-b21a-570a061c5e32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:31:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:02Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab776d02de6 namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: f48a5e60-6b1c-4a3f-91d3-fa035511d3a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:31:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:40:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:18Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3ab7963cb829 namespace: kserve-ci-e2e-test resourceVersion: "30397" uid: e7f3d620-e812-4817-b3ef-4a379449c99f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3b385fcebca1 namespace: kserve-ci-e2e-test resourceVersion: "30355" uid: 967a8480-7713-416a-b7d5-0c5c0eda2db9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:40:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:15Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3b385fcf5bff namespace: kserve-ci-e2e-test resourceVersion: "30356" uid: 99cf5c4f-adc7-4042-ba5b-30bf4b959af4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 416f5e13-7be6-4ebf-a032-2295aca3c479 kind: Event lastTimestamp: "2026-07-01T17:40:18Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:40:18Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn.18be3b390abf31c4 namespace: kserve-ci-e2e-test resourceVersion: "30395" uid: 97fbfada-fe23-4423-a738-40901efa729b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bbf0f-predictor-6d7cf8c5c namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: ec415dda-4859-406a-b463-ae8e004ca69b kind: Event lastTimestamp: "2026-07-01T17:31:00Z" message: 'Created pod: success-200-isvc-bbf0f-predictor-6d7cf8c5c-bc4gn' metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:00Z" name: success-200-isvc-bbf0f-predictor-6d7cf8c5c.18be3ab72ed78fec namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 8c927e0e-14ab-4d3c-b2e6-bbf46f7222f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: 1f154bd6-c9ba-488c-a66a-c5ff178eb209 kind: Event lastTimestamp: "2026-07-01T17:31:00Z" message: Scaled up replica set success-200-isvc-bbf0f-predictor-6d7cf8c5c from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:00Z" name: success-200-isvc-bbf0f-predictor.18be3ab72c30fe38 namespace: kserve-ci-e2e-test resourceVersion: "26223" uid: b31df215-10e8-42d0-b9eb-061d741ac050 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 31d2729b-a958-4800-ac5a-e848cdd7520a kind: Event lastTimestamp: "2026-07-01T17:31:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:15Z" name: success-200-isvc-bbf0f-predictor.18be3abaab0f27d4 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 1ae991ec-7bda-43e7-8fb6-7febbbf3964d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 31d2729b-a958-4800-ac5a-e848cdd7520a kind: Event lastTimestamp: "2026-07-01T17:31:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:15Z" name: success-200-isvc-bbf0f-predictor.18be3abaab6d5d16 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 578de5ca-117a-4d6c-9e66-db55ed5c6d5d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:31:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 31d2729b-a958-4800-ac5a-e848cdd7520a kind: Event lastTimestamp: "2026-07-01T17:32:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:15Z" name: success-200-isvc-bbf0f-predictor.18be3abe2a259221 namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: 4a46b82e-a811-4c8f-b00b-bd1a4d644560 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T17:31:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bbf0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 31d2729b-a958-4800-ac5a-e848cdd7520a kind: Event lastTimestamp: "2026-07-01T17:32:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:32:15Z" name: success-200-isvc-bbf0f-predictor.18be3abe2a8f89dc namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 9fd1f786-3456-4815-a9de-6d685b633313 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:31:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: 84efd4ba-7ba7-4213-8c43-87fc7fd9c932 kind: Event lastTimestamp: "2026-07-01T17:31:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:31:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:48Z" name: success-200-isvc-bbf0f.18be3ab72c49d2fe namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: f55b8b72-d1ab-4359-84b9-ac5e0e135be4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bbf0f namespace: kserve-ci-e2e-test resourceVersion: "26778" uid: 84efd4ba-7ba7-4213-8c43-87fc7fd9c932 kind: Event lastTimestamp: "2026-07-01T17:31:48Z" message: InferenceService [success-200-isvc-bbf0f] is Ready metadata: creationTimestamp: "2026-07-01T17:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:48Z" name: success-200-isvc-bbf0f.18be3ac2548903f9 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: b71b9053-f43c-43b8-9d07-b7df41b4e00b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:10:16.537810Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ce016-predictor-6ffdb68b79-hq49w to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:10:16Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995739c1738 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: ee0921c0-3017-4ebd-b481-72ab5578db92 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ce016-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995823908fb namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 97fd2b59-786f-43c2-8288-c1ca0f2d3718 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b16e009a namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: d3db69e8-09e7-4530-8b41-915d9ac7fc0f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b2c7a833 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: ce99440d-4542-4d6d-b93e-31c03895abae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b5aaa0d3 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: cb227ba8-f61e-4104-926f-0aa93f16f7ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b623b02b namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 90d83172-d057-4a48-8ae9-2b7984742461 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b62e3e3b namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 513c3f7e-8d34-4a0b-8552-1e840919b7c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b8c4f9fe namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: b24fd704-ad2b-44b5-87e6-c0e03f2ccf81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:10:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:17Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3995b93d720f namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 26a871e5-d0a8-4da3-9b86-aaa774439c22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:10:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:10:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:55Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be399627f6036c namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 8f9d6edc-7446-4960-96a5-4730ca5b6999 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3a16a145b0c8 namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: 646af587-4e7c-4f58-aa74-3e90b19e437f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: bee5b713-8b32-437d-86ad-c6c646b805c3 kind: Event lastTimestamp: "2026-07-01T17:19:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:31Z" name: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w.18be3a16a1465b6f namespace: kserve-ci-e2e-test resourceVersion: "19966" uid: 0d50fc3a-718e-4673-b347-a90de4438897 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ce016-predictor-6ffdb68b79 namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: ee8e0606-3fba-4da1-be96-56bfaba1ed49 kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: 'Created pod: success-200-isvc-ce016-predictor-6ffdb68b79-hq49w' metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:16Z" name: success-200-isvc-ce016-predictor-6ffdb68b79.18be399572ffd7a0 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 249bc3c2-2c5c-42b4-97fd-926d2bd447e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: c81d7ce4-963a-454e-9437-2c49212664dd kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: Scaled up replica set success-200-isvc-ce016-predictor-6ffdb68b79 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:10:16Z" name: success-200-isvc-ce016-predictor.18be399570488f05 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: d3c9c902-e101-4695-9039-fabe946c3a19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:10:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: d63b60bf-040f-4ac6-b242-7196b5ab7099 kind: Event lastTimestamp: "2026-07-01T17:11:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:01Z" name: success-200-isvc-ce016-predictor.18be3998eea75c15 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: e37680cc-6382-4770-88c2-ec61125653b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:10:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: d63b60bf-040f-4ac6-b242-7196b5ab7099 kind: Event lastTimestamp: "2026-07-01T17:11:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:01Z" name: success-200-isvc-ce016-predictor.18be3998eee7819f namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 8ed8da41-e88d-4455-b7ff-c17612fe39b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: d63b60bf-040f-4ac6-b242-7196b5ab7099 kind: Event lastTimestamp: "2026-07-01T17:11:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:31Z" name: success-200-isvc-ce016-predictor.18be399c6deb5cd0 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 45b2fdf6-4b2e-4995-91ad-baa7472122cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce016-predictor namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: d63b60bf-040f-4ac6-b242-7196b5ab7099 kind: Event lastTimestamp: "2026-07-01T17:11:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:31Z" name: success-200-isvc-ce016-predictor.18be399c6e5c6a22 namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 15fdd60d-9841-4241-a65c-ebd2090b3b6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:10:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce016 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 64c2a329-8672-4e6b-8a79-a4ca4db47d53 kind: Event lastTimestamp: "2026-07-01T17:11:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:05Z" name: success-200-isvc-ce016.18be39957066ba64 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 2a19ca32-1cc2-4e19-b7e3-5800c6e19374 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce016 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 64c2a329-8672-4e6b-8a79-a4ca4db47d53 kind: Event lastTimestamp: "2026-07-01T17:11:05Z" message: InferenceService [success-200-isvc-ce016] is Ready metadata: creationTimestamp: "2026-07-01T17:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:05Z" name: success-200-isvc-ce016.18be39a0e325655d namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 44e66235-a5eb-4184-ab8b-ade714ad1e52 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:30:06.501864Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25464" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-db836-predictor-598457c8f9-bc6fv to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaa82feb0fa namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: dedcacd2-03c7-4781-91f7-075a02023e83 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-db836-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaa8ca29824 namespace: kserve-ci-e2e-test resourceVersion: "25504" uid: 42a56e61-f7ed-4ffe-81c2-750cfde6b528 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25475" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaac12d83f2 namespace: kserve-ci-e2e-test resourceVersion: "25555" uid: b407393b-f611-481c-a60b-5e3d2d43523e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaac281396e namespace: kserve-ci-e2e-test resourceVersion: "25557" uid: d7cde4ac-6adc-4619-a1ed-307844f9e29c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaac5e8ef16 namespace: kserve-ci-e2e-test resourceVersion: "25559" uid: a11280de-3691-4c14-aada-d587ab97e520 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaac66425a6 namespace: kserve-ci-e2e-test resourceVersion: "25560" uid: b66d26bb-446f-4d0d-9b71-67220198c8ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaac66dc96b namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: dc7ea9c2-adca-4551-91b5-93ecfbd977c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaac99f0899 namespace: kserve-ci-e2e-test resourceVersion: "25564" uid: 5e0736cf-4d09-44d4-afd2-e94fce1fce17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:07Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaaca15c6c4 namespace: kserve-ci-e2e-test resourceVersion: "25565" uid: 22d2e8c9-3fca-4a10-a198-87c15d182a75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T17:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:30:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T17:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:30:45Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3aaaf5531b78 namespace: kserve-ci-e2e-test resourceVersion: "26014" uid: 950f1f8e-a065-4615-a055-dd85cd90a45b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:47:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:26Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3b9c8c2be287 namespace: kserve-ci-e2e-test resourceVersion: "33348" uid: ce07d12c-b7e3-441f-a823-3984b752fde1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db836-predictor-598457c8f9-bc6fv namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0b838ab0-36ca-443c-a390-0323761a03fc kind: Event lastTimestamp: "2026-07-01T17:47:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T17:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:26Z" name: success-200-isvc-db836-predictor-598457c8f9-bc6fv.18be3b9c8c2c83a1 namespace: kserve-ci-e2e-test resourceVersion: "33349" uid: 35b8808c-c45b-43c2-9845-2bf22f360428 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-db836-predictor-598457c8f9 namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: f328db89-23fd-44d4-8fdd-d7d63579ceb3 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'Created pod: success-200-isvc-db836-predictor-598457c8f9-bc6fv' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-db836-predictor-598457c8f9.18be3aaa82874747 namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: e5ae9c7e-d193-4bdf-8e34-d43695c05861 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25442" uid: 459b5374-69f8-4291-a6e3-b8a9c0363e4e kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: Scaled up replica set success-200-isvc-db836-predictor-598457c8f9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-db836-predictor.18be3aaa7de27336 namespace: kserve-ci-e2e-test resourceVersion: "25452" uid: 653dd888-d2d8-4e35-acaa-34b3388d023b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25445" uid: 01e697e5-e7ec-4d02-b261-87336de688bd kind: Event lastTimestamp: "2026-07-01T17:30:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:36Z" name: success-200-isvc-db836-predictor.18be3aadfc6afe8e namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 31012ca1-36bb-4953-893b-795c542c7be1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T17:30:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25445" uid: 01e697e5-e7ec-4d02-b261-87336de688bd kind: Event lastTimestamp: "2026-07-01T17:30:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:30:36Z" name: success-200-isvc-db836-predictor.18be3aadfcd102c6 namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: f39fe3fa-3619-454f-8133-e39f2cdb3471 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 01e697e5-e7ec-4d02-b261-87336de688bd kind: Event lastTimestamp: "2026-07-01T17:31:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:21Z" name: success-200-isvc-db836-predictor.18be3ab4faef74b2 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 81e713cc-91d2-4971-a32e-081e7d4d9de4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db836-predictor namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 01e697e5-e7ec-4d02-b261-87336de688bd kind: Event lastTimestamp: "2026-07-01T17:31:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:21Z" name: success-200-isvc-db836-predictor.18be3ab4fb795b41 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 9dbe7e49-0939-449a-b3b2-0828a2b87083 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "25415" uid: 46f8e2ea-6572-4088-a08b-2cce51b4fb36 kind: Event lastTimestamp: "2026-07-01T17:30:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:55Z" name: success-200-isvc-db836.18be3aaa7ddea450 namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: 57cdee7e-fe65-44b9-8fdd-074044e412a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "25415" uid: 46f8e2ea-6572-4088-a08b-2cce51b4fb36 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-db836": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db836": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-db836.18be3aaa7f26d0a5 namespace: kserve-ci-e2e-test resourceVersion: "25459" uid: c21d9e8c-8592-4435-aeb2-3f4875f4b2cb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "25415" uid: 46f8e2ea-6572-4088-a08b-2cce51b4fb36 kind: Event lastTimestamp: "2026-07-01T17:30:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db836": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T17:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:06Z" name: success-200-isvc-db836.18be3aaa7f2710a5 namespace: kserve-ci-e2e-test resourceVersion: "25462" uid: 212e484a-3e70-447a-8ed1-847fb53ec0f7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db836 namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 46f8e2ea-6572-4088-a08b-2cce51b4fb36 kind: Event lastTimestamp: "2026-07-01T17:30:55Z" message: InferenceService [success-200-isvc-db836] is Ready metadata: creationTimestamp: "2026-07-01T17:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:30:55Z" name: success-200-isvc-db836.18be3ab5eed70ce9 namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 503571c0-06d2-4bfa-88df-dc447bc0f239 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:09:36.169831Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1b15a-5bf689b779-zll7z to ip-10-0-137-180.ec2.internal metadata: creationTimestamp: "2026-07-01T17:09:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:09:36Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398c0d7db69c namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: c5669837-6bb5-41bc-9fcb-5303083e1623 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:36Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1b15a-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:09:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:36Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398c1e12345b namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 13fb1ad9-26b6-4e74-b504-a18f5b9ac870 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:37Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:09:37Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398c4ac022ea namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: b4afa89a-d8f7-44d6-b3ba-c8a2c75261a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b15a} kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:37Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" metadata: creationTimestamp: "2026-07-01T17:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:37Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398c4c34aaf5 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 1bff787c-4b56-4cbb-abbc-bf0ae39458a6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b15a} kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" in 2.617s (2.617s including waiting). Image size: 218070140 bytes.' metadata: creationTimestamp: "2026-07-01T17:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:39Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398ce83d35bb namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 5f08e61a-4ad5-45dc-b8b5-e5bcb70eb0f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b15a} kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:39Z" message: 'Created container: switch-graph-1b15a' metadata: creationTimestamp: "2026-07-01T17:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:39Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398ceb9c75b1 namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: db89cf93-52ef-46ff-95c7-e19c8cd6bf16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b15a} kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:39Z" message: Started container switch-graph-1b15a metadata: creationTimestamp: "2026-07-01T17:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:39Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398cec134d39 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: d6bda456-11d0-4fa7-8c32-977e7fa0a906 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b15a} kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:09:50Z" message: Stopping container switch-graph-1b15a metadata: creationTimestamp: "2026-07-01T17:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:09:50Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398f5cfac022 namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: e9101a39-4336-41b3-b8ad-896f2bb294ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b15a} kind: Pod name: switch-graph-1b15a-5bf689b779-zll7z namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: d52c91d6-3f99-4d01-9226-ce261eb999b1 kind: Event lastTimestamp: "2026-07-01T17:10:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:10:16Z" name: switch-graph-1b15a-5bf689b779-zll7z.18be398f98e9f1aa namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: fa37d0f5-3700-4606-b03a-45d4c7225b4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-180.ec2.internal source: component: kubelet host: ip-10-0-137-180.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1b15a-5bf689b779 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 1621a9c3-6b68-4c74-98ad-d61dc9d15fc7 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: 'Created pod: switch-graph-1b15a-5bf689b779-zll7z' metadata: creationTimestamp: "2026-07-01T17:09:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:36Z" name: switch-graph-1b15a-5bf689b779.18be398c0d1be761 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: b8fa4a09-663a-462a-bfa8-e2211343eccb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1b15a namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 8ffb2c56-3b61-425c-b3a8-c9b60ae6cb69 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: Scaled up replica set switch-graph-1b15a-5bf689b779 from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:09:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:09:36Z" name: switch-graph-1b15a.18be398c0be32b20 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 04574776-b4aa-422e-8ec2-06dea09a8596 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:09:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1b15a namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 72dded09-a740-4b17-a167-83a78d70a1b6 kind: Event lastTimestamp: "2026-07-01T17:09:36Z" message: InferenceGraph [switch-graph-1b15a] is Ready metadata: creationTimestamp: "2026-07-01T17:09:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:09:36Z" name: switch-graph-1b15a.18be398c10c997e3 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 447723e4-1dbd-48e2-8216-03d4af1f5db4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:11:00.616676Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3c336-67f978898d-48sxc to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:11:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:11:00Z" name: switch-graph-3c336-67f978898d-48sxc.18be399fb6ea5072 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: ce130907-b041-4c62-81ee-bdfef3a9e2a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:11:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3c336-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T17:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:00Z" name: switch-graph-3c336-67f978898d-48sxc.18be399fc3e4d9f4 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: dd7aca33-e2da-4ba3-8e41-9d25cebf3071 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:11:01Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:11:01Z" name: switch-graph-3c336-67f978898d-48sxc.18be399ff480c40a namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: f41afb06-e65b-4e63-b403-785dc9b0be71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c336} kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:11:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:01Z" name: switch-graph-3c336-67f978898d-48sxc.18be399ff5f77dc5 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 40804cb2-cd86-42c1-a6ca-9c2f3f992e20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c336} kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:11:01Z" message: 'Created container: switch-graph-3c336' metadata: creationTimestamp: "2026-07-01T17:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:01Z" name: switch-graph-3c336-67f978898d-48sxc.18be399ff8f77822 namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 58de0a9b-ea5e-4bd1-a287-52a28dda208d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c336} kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:11:01Z" message: Started container switch-graph-3c336 metadata: creationTimestamp: "2026-07-01T17:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:11:01Z" name: switch-graph-3c336-67f978898d-48sxc.18be399ff9b16425 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 2f638c36-39ac-4923-919c-83c928d7d3a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c336} kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:19:15Z" message: Stopping container switch-graph-3c336 metadata: creationTimestamp: "2026-07-01T17:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:15Z" name: switch-graph-3c336-67f978898d-48sxc.18be3a12e53886c4 namespace: kserve-ci-e2e-test resourceVersion: "19655" uid: a6a9f7f4-2491-42ce-8553-c1e3e40415fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:19:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3c336} kind: Pod name: switch-graph-3c336-67f978898d-48sxc namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 2f954d33-263e-402e-87e2-24efc5c36fe2 kind: Event lastTimestamp: "2026-07-01T17:19:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:19:43Z" name: switch-graph-3c336-67f978898d-48sxc.18be3a1390c838c7 namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: f995d371-4bdc-463d-9584-4658bd53762e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3c336-67f978898d namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: d8ede9bb-7982-454e-9a38-467872d8d6b6 kind: Event lastTimestamp: "2026-07-01T17:11:00Z" message: 'Created pod: switch-graph-3c336-67f978898d-48sxc' metadata: creationTimestamp: "2026-07-01T17:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:00Z" name: switch-graph-3c336-67f978898d.18be399fb6843c0f namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 6230ef63-bac8-44db-9f23-82018ef237d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3c336 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d3ab2412-e9dd-4396-91dd-ecc18709d9aa kind: Event lastTimestamp: "2026-07-01T17:11:00Z" message: Scaled up replica set switch-graph-3c336-67f978898d from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:00Z" name: switch-graph-3c336.18be399fb54d1004 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 372e76a8-a4e1-4053-9c02-743a82b92af7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3c336 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 59c672c8-2dca-47f8-b6b2-a438e8a17c67 kind: Event lastTimestamp: "2026-07-01T17:11:00Z" message: InferenceGraph [switch-graph-3c336] is Ready metadata: creationTimestamp: "2026-07-01T17:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:11:00Z" name: switch-graph-3c336.18be399fbb7f39dc namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 1a626d0b-0aeb-4b21-846d-d5194dd596c7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3c336 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 017ff146-dd9e-4ce7-a8f7-25db7bf0f2e6 kind: Event lastTimestamp: "2026-07-01T17:11:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:45Z" name: switch-graph-3c336.18be39a3343c4cd2 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 4d8ef654-ffe4-46b5-b6e9-047c9582c9d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T17:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3c336 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 017ff146-dd9e-4ce7-a8f7-25db7bf0f2e6 kind: Event lastTimestamp: "2026-07-01T17:11:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:11:45Z" name: switch-graph-3c336.18be39a334c6e610 namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: b3ebc86b-0c30-4f40-a0bf-3f9f32ed17a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T17:31:06.478581Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-db836-5758686fdc-568g5 to ip-10-0-138-165.ec2.internal metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836-5758686fdc-568g5.18be3ab879e2c41d namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: 63bf09dc-81ef-43b8-9140-bde0ed1a0352 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79fb68874c-62mj8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836-5758686fdc-568g5.18be3ab8938d16e7 namespace: kserve-ci-e2e-test resourceVersion: "26421" uid: b3c9ec2f-4cf0-4347-993f-7c92473c508f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-db836} kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c0785f60b462f861cfab04ad440fa6adb06dbacec8c1d6b98ebf808ae746d9cf" already present on machine metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836-5758686fdc-568g5.18be3ab894d6a889 namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 5c2875ec-8c0a-442c-a8e8-c56eb294cf61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-db836} kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: 'Created container: switch-graph-db836' metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836-5758686fdc-568g5.18be3ab897b39898 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 4f38e2d9-a6e8-4af9-97b4-c2f675902f02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-db836} kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: Started container switch-graph-db836 metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836-5758686fdc-568g5.18be3ab89828e49d namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: f61b58ee-65fa-479b-a398-61dfceae5e7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-db836} kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: "2026-07-01T17:47:25Z" message: Stopping container switch-graph-db836 metadata: creationTimestamp: "2026-07-01T17:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:25Z" name: switch-graph-db836-5758686fdc-568g5.18be3b9c7d575c34 namespace: kserve-ci-e2e-test resourceVersion: "33328" uid: 54344200-dd35-4f62-a928-abe33bf2d266 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T17:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-db836} kind: Pod name: switch-graph-db836-5758686fdc-568g5 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 7e8ea489-c354-493a-ae8c-c376ed1547be kind: Event lastTimestamp: "2026-07-01T17:47:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T17:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T17:47:53Z" name: switch-graph-db836-5758686fdc-568g5.18be3b9d1e890c2a namespace: kserve-ci-e2e-test resourceVersion: "33601" uid: b6930ce5-c8ce-469f-8c15-f66b641c9349 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-165.ec2.internal source: component: kubelet host: ip-10-0-138-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-db836-5758686fdc namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 7ecd8f6b-d0e7-41d2-a125-922cc1dc57ef kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: 'Created pod: switch-graph-db836-5758686fdc-568g5' metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836-5758686fdc.18be3ab879614f2d namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 08de5ad2-f243-4bd2-bf68-211ad329647c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-db836 namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: bdf14496-6308-44b0-b7fb-bf0cbc1c9ca9 kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: Scaled up replica set switch-graph-db836-5758686fdc from 0 to 1 metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836.18be3ab878a5c655 namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: 3e7daff1-96b9-4655-ad9a-02067492c2a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-db836 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: 337219e4-7768-490b-a054-c21c844c45a6 kind: Event lastTimestamp: "2026-07-01T17:31:06Z" message: InferenceGraph [switch-graph-db836] is Ready metadata: creationTimestamp: "2026-07-01T17:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T17:31:06Z" name: switch-graph-db836.18be3ab87f71921d namespace: kserve-ci-e2e-test resourceVersion: "26416" uid: 11b84c8c-2a1c-486a-8904-b75318917589 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-db836 namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 72ce316a-6a03-4e6c-bfb7-0167b9cbf7e3 kind: Event lastTimestamp: "2026-07-01T17:31:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:21Z" name: switch-graph-db836.18be3abbf6f3a48b namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: f5ceb9d7-e657-4af5-9038-fd0bc1e6d54c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-db836 namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: 72ce316a-6a03-4e6c-bfb7-0167b9cbf7e3 kind: Event lastTimestamp: "2026-07-01T17:31:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T17:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:21Z" name: switch-graph-db836.18be3abbf75f1140 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 298df1dc-fc20-4b5d-aa77-61454e927905 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-db836 namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 72ce316a-6a03-4e6c-bfb7-0167b9cbf7e3 kind: Event lastTimestamp: "2026-07-01T17:31:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:36Z" name: switch-graph-db836.18be3abf762142fd namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: abd6267a-9510-4655-b140-3a4632a21a6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T17:31:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-db836 namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 72ce316a-6a03-4e6c-bfb7-0167b9cbf7e3 kind: Event lastTimestamp: "2026-07-01T17:31:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T17:31:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T17:31:36Z" name: switch-graph-db836.18be3abf768a4cbf namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 485dd48c-6ba1-46bf-bb36-78a01095a4b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33605"