--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:00.049906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-305f4-d67d47567-krvxn to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:00Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfcee992f0155c namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: fa04fbd8-b155-4e61-bc7c-9fd63e886937 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-305f4-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:00Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfcee99efa0cd9 namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: bf094062-0742-4a5f-be2f-3293b6af197b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:01Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:01Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfceea05bc9397 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 9a34de8a-9fe0-4395-b61f-8891616d54de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-305f4} kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:02Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfceea071f8637 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 535d751c-11cf-4d5d-ad7e-a72860bc5a21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-305f4} kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:02Z" message: 'Created container: ensemble-graph-305f4' metadata: creationTimestamp: "2026-07-06T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:02Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfceea09d44aa5 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 5a4345a9-4420-48ae-b874-c2ff50fdbda1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-305f4} kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:02Z" message: Started container ensemble-graph-305f4 metadata: creationTimestamp: "2026-07-06T20:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:02Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfceea0a3db7a4 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: b2d6fec4-bed5-423b-9064-ff297ebfc707 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-305f4} kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Stopping container ensemble-graph-305f4 metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfceebe9db3b3f namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 70b55131-feac-4639-a7ea-9ce7946f60b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-305f4} kind: Pod name: ensemble-graph-305f4-d67d47567-krvxn namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: b2eccb3f-95f3-4ffb-aeb4-4604950b7089 kind: Event lastTimestamp: "2026-07-06T20:58:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:38Z" name: ensemble-graph-305f4-d67d47567-krvxn.18bfceecb6d7e387 namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 1e65c492-1f3e-4b0a-a69e-c53c165de0f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-305f4-d67d47567 namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 305e4f14-90ae-40d6-a211-f33a0f6a0ea2 kind: Event lastTimestamp: "2026-07-06T20:58:00Z" message: 'Created pod: ensemble-graph-305f4-d67d47567-krvxn' metadata: creationTimestamp: "2026-07-06T20:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:00Z" name: ensemble-graph-305f4-d67d47567.18bfcee99247c7fd namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: da1857ab-54f6-41ea-bad8-97571e98e736 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-305f4 namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: ad27da96-51ea-48ba-8b84-c9d0fed58e17 kind: Event lastTimestamp: "2026-07-06T20:58:00Z" message: Scaled up replica set ensemble-graph-305f4-d67d47567 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:00Z" name: ensemble-graph-305f4.18bfcee990d95a10 namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: f54ce6ef-9a32-4b6e-8d76-267ee5f667ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-305f4 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: ecfd62f3-e12a-49db-8de8-96847da25358 kind: Event lastTimestamp: "2026-07-06T20:58:00Z" message: InferenceGraph [ensemble-graph-305f4] is Ready metadata: creationTimestamp: "2026-07-06T20:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:00Z" name: ensemble-graph-305f4.18bfcee99790a34f namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 1886985c-6c46-455e-be77-79489b8d2911 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:59:10.350981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-858c3-7798d67676-c5m7h to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:59:10Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcef9f1354bbb namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: c59af5fb-c0d0-44af-a826-3eb51c366a99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-858c3-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:10Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcef9faf968ae namespace: kserve-ci-e2e-test resourceVersion: "21802" uid: 14586e6e-d45f-4a5c-8fb4-9f30fa6a37c8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:11Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T20:59:11Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:59:11Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcefa2e2a6ecd namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: dad95301-c1b2-45cd-88ed-e5e9b889642e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-858c3} kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T20:59:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:11Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcefa2fe3fd7b namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 9c3d6178-ca17-4522-a79e-6a3917d227e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-858c3} kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T20:59:11Z" message: 'Created container: ensemble-graph-858c3' metadata: creationTimestamp: "2026-07-06T20:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:11Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcefa32833f54 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: 3c341002-eb04-4e52-9da2-a412cfa5f280 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-858c3} kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T20:59:11Z" message: Started container ensemble-graph-858c3 metadata: creationTimestamp: "2026-07-06T20:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:11Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcefa32eea199 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: f80a12fa-aef4-4e45-a15a-57185007c68f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-858c3} kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T21:07:24Z" message: Stopping container ensemble-graph-858c3 metadata: creationTimestamp: "2026-07-06T21:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:24Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcf6d15153438 namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: 45b79f0f-a8ae-4593-95c9-a92a1a237462 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-858c3} kind: Pod name: ensemble-graph-858c3-7798d67676-c5m7h namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: f2d1ae92-8e3f-4906-bdd7-f5a94ed1e828 kind: Event lastTimestamp: "2026-07-06T21:07:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:52Z" name: ensemble-graph-858c3-7798d67676-c5m7h.18bfcf6dc3272cab namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: efa9a473-d656-4c6c-a20f-21bacea14d0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-858c3-7798d67676 namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: a4b131c5-0b02-4257-86ed-68e1158e3132 kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'Created pod: ensemble-graph-858c3-7798d67676-c5m7h' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: ensemble-graph-858c3-7798d67676.18bfcef9f074e3c6 namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 157a0d83-ac39-4966-b3ca-7087196aef15 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-858c3 namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 99c1c901-ccbf-43a7-9f6c-d5629566b48f kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: Scaled up replica set ensemble-graph-858c3-7798d67676 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: ensemble-graph-858c3.18bfcef9ef7290de namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: f6c2fad1-2937-46c7-8913-42cb4a5d4c09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-858c3 namespace: kserve-ci-e2e-test resourceVersion: "21792" uid: 5cc62c4f-0077-4642-96aa-3bd1d368104f kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: InferenceGraph [ensemble-graph-858c3] is Ready metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:10Z" name: ensemble-graph-858c3.18bfcef9f3b1ed21 namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 6bc6898b-6ddf-47c8-8330-e855cafd63ac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:59:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-858c3 namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: dd07431b-e8df-43be-9ea2-d7453404d6d6 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:40Z" name: ensemble-graph-858c3.18bfcefd6e1f8230 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: 9b1d6ab2-a35d-4dd5-b99f-28ed07dc2268 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:59:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-858c3 namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: dd07431b-e8df-43be-9ea2-d7453404d6d6 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:40Z" name: ensemble-graph-858c3.18bfcefd6e6def6e namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: b73c21a8-afa8-43df-a7f7-0be373ae5af6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:57:30.074369Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:57:30Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee29841b7ce namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: ef8b3774-14db-41bb-9ae9-01dade56d168 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-24ddf-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2a774e3d7 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 610a3aaf-30f8-4514-bfdd-b4504adb21d7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2d5d1aece namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: e0a30b5f-a694-491b-a045-a4c6da022b30 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2d7706bc4 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: d753a621-8485-44b3-9328-148563bf87d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2dacb2c89 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 9dc45e35-07dc-49a2-957a-0fc12af91f1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2db3dc0f5 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 7c5029c1-36af-4b9a-a92c-543eda998bef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2db4807ee namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: a5e4d4de-f0b2-4538-9b68-6d6c6f670d08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2de4c46cf namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 3fa2bbc4-dd96-4508-98c5-fa58fc5fd88f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee2dec1ba3e namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 3ac9f6dc-11f6-4b05-8076-7d04edd0c13c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:58:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:08Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcee325230ef5 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 1fe2041f-b6e4-415f-aa46-f15f6fe1b8a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcef2f1f1a048 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 6c0c5400-3196-4b7d-a4d2-e0225a055926 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcef2f1f23fce namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 119b652c-62ec-446f-b979-dc66aef5427a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 58229cdc-1038-4689-96c7-428db5647f04 kind: Event lastTimestamp: "2026-07-06T20:58:43Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:43Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw.18bfcef3ad5b89c7 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: c2aa23ac-5d43-488a-bfbe-cdba306918d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-24ddf-predictor-665cbb8c8f namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: cdb940f7-3857-4d2e-b3e6-0d14d3884790 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: 'Created pod: error-404-isvc-24ddf-predictor-665cbb8c8f-25wjw' metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:30Z" name: error-404-isvc-24ddf-predictor-665cbb8c8f.18bfcee2977e4e12 namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: 869eebf6-d9bd-4b36-a82a-d1c72d346311 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: fc0dbe08-3b51-4cd8-ac75-b6042c468e1f kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Scaled up replica set error-404-isvc-24ddf-predictor-665cbb8c8f from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:29Z" name: error-404-isvc-24ddf-predictor.18bfcee292f583bf namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 19e149d9-d73f-432b-a683-7b052ff4ed39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 427843e5-8073-42da-be14-4774708e7190 kind: Event lastTimestamp: "2026-07-06T20:58:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:00Z" name: error-404-isvc-24ddf-predictor.18bfcee610fb6372 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: dee2101a-af66-4e3d-9508-538c4ad6ed7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 427843e5-8073-42da-be14-4774708e7190 kind: Event lastTimestamp: "2026-07-06T20:58:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:00Z" name: error-404-isvc-24ddf-predictor.18bfcee6119457b5 namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 2f52a8cc-f06f-4456-ad15-225452e242d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: 427843e5-8073-42da-be14-4774708e7190 kind: Event lastTimestamp: "2026-07-06T20:58:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:30Z" name: error-404-isvc-24ddf-predictor.18bfceed0f1632cd namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 103711d0-8f19-4f59-b472-0afbe5e2c256 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: 427843e5-8073-42da-be14-4774708e7190 kind: Event lastTimestamp: "2026-07-06T20:58:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:30Z" name: error-404-isvc-24ddf-predictor.18bfceed0f8d3268 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 64d0493e-281f-4ed8-a09e-026acf250aa4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24ddf namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 87020785-dd3c-43ae-99f7-40e5819b4c57 kind: Event lastTimestamp: "2026-07-06T20:58:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:18Z" name: error-404-isvc-24ddf.18bfcee29330c7fc namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: d884bb6c-91f5-4085-8e9a-c45ca587d617 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24ddf namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 87020785-dd3c-43ae-99f7-40e5819b4c57 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-24ddf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-24ddf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:57:30Z" name: error-404-isvc-24ddf.18bfcee2950dac87 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: df11f48b-9520-4adc-99a5-d85efb86552d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24ddf namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 87020785-dd3c-43ae-99f7-40e5819b4c57 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-24ddf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:57:30Z" name: error-404-isvc-24ddf.18bfcee2950de9c6 namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 591eb691-fac9-4796-b440-aec83ca12a98 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24ddf namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 87020785-dd3c-43ae-99f7-40e5819b4c57 kind: Event lastTimestamp: "2026-07-06T20:58:18Z" message: InferenceService [error-404-isvc-24ddf] is Ready metadata: creationTimestamp: "2026-07-06T20:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:18Z" name: error-404-isvc-24ddf.18bfceede18d8d78 namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: f6bf62c2-a870-46d7-b183-88718011e108 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:56:50.074676Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced94816d516 namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 60eba404-a3f7-448a-bf72-c3b1483103f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-305f4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced954c98fab namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 4d39f568-e218-44f6-bb37-d1dcecb55d6d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced9869505c9 namespace: kserve-ci-e2e-test resourceVersion: "20069" uid: d6e43874-b363-4970-84d3-eb5d1f51f22b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced988493bbf namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: 9f587043-8733-4809-9980-c634d10d41f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced98a7db4be namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 38772538-2a80-470e-9bd0-beedbd358128 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced98aeb2e09 namespace: kserve-ci-e2e-test resourceVersion: "20077" uid: 503b1ed5-a857-47bb-b76e-16f2506eb4e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced98af48e71 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 8812a798-00b0-4cde-8a74-ea28e4dcb034 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced98e37a9c4 namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: fe87bb5d-880f-4f5a-9831-7f28d5675084 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced98eb044a2 namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 91f94e99-f226-467f-aa1e-8708a510a05c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:57:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:28Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfced9c8e25fe4 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 902fcf46-7e46-4b48-b572-45361b1627a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfceebfe735bcb namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: d260f111-50dd-47c1-a6db-ae08a12a8889 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfceebfe793858 namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: 6fc8e5a6-3776-4de7-9ce4-ce33db856be6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 5d6f37bd-f266-4ce2-b814-c84d82b52b2a kind: Event lastTimestamp: "2026-07-06T20:58:13Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:13Z" name: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2.18bfceeca4d4fb86 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 8e122284-491f-47ba-9545-c845c5e5e642 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-305f4-predictor-5d5987d4f namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: c1fa75d8-0f1d-4a6a-beb1-d041c23c11b1 kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: 'Created pod: error-404-isvc-305f4-predictor-5d5987d4f-pz2z2' metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-305f4-predictor-5d5987d4f.18bfced9479a5d2f namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 4a0ee336-4789-4bbe-8a9e-2308bd8e98a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 8232bd50-f1a0-4938-b66a-588bca172c2f kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: Scaled up replica set error-404-isvc-305f4-predictor-5d5987d4f from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-305f4-predictor.18bfced945bd7a6d namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: 78aa0f43-db7a-49f5-afc1-c49c1d6bb787 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: fa5eea8b-5ce2-4bc5-a747-c2669c4a572f kind: Event lastTimestamp: "2026-07-06T20:57:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:05Z" name: error-404-isvc-305f4-predictor.18bfcedcc431c8ab namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 5a895db5-26fc-4217-aa8a-ac2b08895906 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: fa5eea8b-5ce2-4bc5-a747-c2669c4a572f kind: Event lastTimestamp: "2026-07-06T20:57:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:05Z" name: error-404-isvc-305f4-predictor.18bfcedcc4a9e476 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: b39efee7-6012-4426-a6bb-08a22998b515 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: fa5eea8b-5ce2-4bc5-a747-c2669c4a572f kind: Event lastTimestamp: "2026-07-06T20:58:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:05Z" name: error-404-isvc-305f4-predictor.18bfcee0436ce18c namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 7d913a93-4fe4-42a5-ae79-afd56ab1b060 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: fa5eea8b-5ce2-4bc5-a747-c2669c4a572f kind: Event lastTimestamp: "2026-07-06T20:58:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:05Z" name: error-404-isvc-305f4-predictor.18bfcee043ede2b5 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 3609bf3b-38ca-4305-8d9f-89378b9c63db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 140d000e-e14a-4911-88da-f7fb0fa1f99b kind: Event lastTimestamp: "2026-07-06T20:57:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:57:38Z" name: error-404-isvc-305f4.18bfced945cc3491 namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 6e214417-fef2-482d-bf16-5b34cd15f2d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 140d000e-e14a-4911-88da-f7fb0fa1f99b kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-305f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-305f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-305f4.18bfced9464cf27f namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: 057f4c1a-f273-44fd-b254-d90647aaa84c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 140d000e-e14a-4911-88da-f7fb0fa1f99b kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-305f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-305f4.18bfced9464d3780 namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: d34e2a4e-4445-4daf-b93e-fbf9a3437639 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 140d000e-e14a-4911-88da-f7fb0fa1f99b kind: Event lastTimestamp: "2026-07-06T20:57:38Z" message: InferenceService [error-404-isvc-305f4] is Ready metadata: creationTimestamp: "2026-07-06T20:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:57:38Z" name: error-404-isvc-305f4.18bfcee48523fb7d namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: ede254a5-6187-4672-ab11-2a725c55b881 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:07:55.151128Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-32e32-predictor-98848c587-cfphf to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf7421bc1633 namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: bf80bd91-78b7-4916-ae36-7a80c38fb97f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf743bff5f13 namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: 1d365f7f-edf2-4244-a654-a25f8c93b7fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf743dbb122b namespace: kserve-ci-e2e-test resourceVersion: "26093" uid: 8a9cb7c9-8514-43e0-bd7b-dbf7251f2719 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf744079cb9b namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: 93e27dca-c57c-406e-8e53-332c5ed879f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf7440e62f68 namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: 9509921f-7bac-47cb-91cb-fce5ca959ead reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf7440efd6f6 namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 8f50de1e-e9bf-4209-9d09-e1288366543f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf74458a2b77 namespace: kserve-ci-e2e-test resourceVersion: "26099" uid: ef3470ec-dcb9-4048-a1e2-52ba6844596a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf7445fd7ea9 namespace: kserve-ci-e2e-test resourceVersion: "26100" uid: 80a397fe-1899-4352-9c8c-7c845c6c3362 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T21:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:08:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:33Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfcf7468e37ce1 namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: 199ef934-2536-44e9-ab52-763a1815b0ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:25:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:25:14Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfd0661564d9d0 namespace: kserve-ci-e2e-test resourceVersion: "33799" uid: 033d539d-bbd9-4a01-8250-50d554701b49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32e32-predictor-98848c587-cfphf namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 29bd7197-a4d2-4c4b-9cbc-83f4a2f2d958 kind: Event lastTimestamp: "2026-07-06T21:25:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:25:14Z" name: error-404-isvc-32e32-predictor-98848c587-cfphf.18bfd06615655247 namespace: kserve-ci-e2e-test resourceVersion: "33800" uid: 11ead044-5308-44be-979d-12a50e930f2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-32e32-predictor-98848c587 namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: bbb83409-0959-4687-ba04-c6f97a988723 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'Created pod: error-404-isvc-32e32-predictor-98848c587-cfphf' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor-98848c587.18bfcf74213ff7f7 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 5cfbd0ef-1fb9-4659-a91d-01bfa8752161 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: 90171f57-deaa-468e-9664-374930de8483 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Scaled up replica set error-404-isvc-32e32-predictor-98848c587 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32-predictor.18bfcf741efde4ef namespace: kserve-ci-e2e-test resourceVersion: "26037" uid: 6379d81d-42d7-47eb-868c-7fc93685e302 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:08:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 8b874078-353f-4d15-9d20-e3241fe83097 kind: Event lastTimestamp: "2026-07-06T21:09:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:10Z" name: error-404-isvc-32e32-predictor.18bfcf779d8eabd1 namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: 9b07a356-26d9-4ec1-af7f-aea2ff367cfb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T21:08:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 8b874078-353f-4d15-9d20-e3241fe83097 kind: Event lastTimestamp: "2026-07-06T21:09:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:10Z" name: error-404-isvc-32e32-predictor.18bfcf779e24e12a namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: b1e4cb46-96eb-4b6a-9491-2ba837485712 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 8b874078-353f-4d15-9d20-e3241fe83097 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: error-404-isvc-32e32-predictor.18bfcf7b1d02f133 namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: ed80da2c-6bb0-4b15-8e23-4d5d62b9b1cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 8b874078-353f-4d15-9d20-e3241fe83097 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: error-404-isvc-32e32-predictor.18bfcf7b1dbd032e namespace: kserve-ci-e2e-test resourceVersion: "26390" uid: 39d9dcd2-51f7-4d16-808f-b4dc59ad16d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: b23f9057-ee59-4a42-8fbf-b2d18c4f8210 kind: Event lastTimestamp: "2026-07-06T21:08:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:43Z" name: error-404-isvc-32e32.18bfcf741f6b6b56 namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 00acd3f2-fa23-4a64-a806-08a87a610cee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: b23f9057-ee59-4a42-8fbf-b2d18c4f8210 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-32e32": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-32e32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32.18bfcf741fd9a7a2 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: 65b4525f-2c0b-4961-a00b-1ce89ae291cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: b23f9057-ee59-4a42-8fbf-b2d18c4f8210 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-32e32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-32e32.18bfcf741fda80fa namespace: kserve-ci-e2e-test resourceVersion: "26041" uid: a7626177-502f-452d-9f7a-c3701936f8fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: b23f9057-ee59-4a42-8fbf-b2d18c4f8210 kind: Event lastTimestamp: "2026-07-06T21:08:43Z" message: InferenceService [error-404-isvc-32e32] is Ready metadata: creationTimestamp: "2026-07-06T21:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:43Z" name: error-404-isvc-32e32.18bfcf7f64ed92f8 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: ba0bc611-1aeb-4a33-acd0-10af7afaf386 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:05.003423Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-85420-predictor-856b7d987-9rsjd to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:05Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce4316d89bac namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: fbf87455-7a99-4d23-af17-55f6ce7c4e0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-85420-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce4320f25cfe namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 73a1b7bb-9579-499d-99c6-b3583bc95a84 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:06Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce43546418cc namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 58fcaa27-cc31-4f63-bf4f-7aea62577714 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:06Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce4355ec6b70 namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: f69d2b61-230e-4357-a748-f94a637ae74f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" in 15.28s (15.28s including waiting). Image size: 1334626402 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce46e4b05ea2 namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: 9e096c00-bf03-41cf-b112-a9fd71ea2472 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce46e87db974 namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: ce8b5c95-9cf8-45d9-96b7-43ed36df31f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce46e8f2e2fb namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 3effd000-4c15-46ad-88d8-bcafd6eaa637 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce46e8fff5bf namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 1ae181c7-193a-430f-8cf9-dda1c7883258 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.558s (2.558s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:23Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce47817a91fb namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 675fef09-edf2-41ab-9ecc-8b9b9aefb353 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:24Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce478506ec5b namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: b3ce525e-cb46-46da-92b3-05ae6463a598 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:46:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:24Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce47858753c5 namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 4e8d7e02-d82b-44e5-904e-790fdb5990ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:47:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:00Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce47987d13ca namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 5ea9da29-3c9f-4ecb-a5c1-2c8302c36f21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce58130d95af namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: efb50681-e6aa-475a-b36e-bd9e94227053 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce58130e21eb namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 13fd695d-6e87-47eb-ae7a-a8f7b366130d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-85420-predictor-856b7d987-9rsjd namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: ae9160a2-4983-46e5-aa8c-c2ae98c0af05 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-85420-predictor-856b7d987-9rsjd.18bfce5820bcaca1 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: c497f5ad-81c9-41db-8fea-aac04f47a197 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-85420-predictor-856b7d987 namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: c1170161-ff7d-4389-b7cf-312f35cc6788 kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: 'Created pod: error-404-isvc-85420-predictor-856b7d987-9rsjd' metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:04Z" name: error-404-isvc-85420-predictor-856b7d987.18bfce43162e9e1c namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 7e194398-7943-4b8a-b756-e6a52ef7c074 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: c9e1cf61-25c9-4095-b564-a29ed88dc072 kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: Scaled up replica set error-404-isvc-85420-predictor-856b7d987 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:04Z" name: error-404-isvc-85420-predictor.18bfce431472e146 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 3bd0b83e-0d95-4aa7-bc70-2b23ae59a49f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 4bf05524-bd83-47e8-9986-d61df6056f42 kind: Event lastTimestamp: "2026-07-06T20:47:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:05Z" name: error-404-isvc-85420-predictor.18bfce469307925f namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 1093aa29-3e8a-4ab3-8a37-b19824b65906 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 4bf05524-bd83-47e8-9986-d61df6056f42 kind: Event lastTimestamp: "2026-07-06T20:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:05Z" name: error-404-isvc-85420-predictor.18bfce469376b492 namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: d66ab4a4-d4fb-4d8f-8307-e0da0b651cf9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 4bf05524-bd83-47e8-9986-d61df6056f42 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-85420-predictor.18bfce548e80b469 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: fc0dce51-4f4f-42f9-8add-407e3383ab96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 4bf05524-bd83-47e8-9986-d61df6056f42 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-85420-predictor.18bfce548ef19c40 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: f65ed9a3-5deb-45f2-9c40-d01ad32ae144 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-85420 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 013ac22a-b94a-49c6-ae7e-149d3c2dc866 kind: Event lastTimestamp: "2026-07-06T20:47:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:10Z" name: error-404-isvc-85420.18bfce43148cc893 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 84a18032-05a0-42f4-930a-1f8b8bda5eaa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-85420 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 013ac22a-b94a-49c6-ae7e-149d3c2dc866 kind: Event lastTimestamp: "2026-07-06T20:47:10Z" message: InferenceService [error-404-isvc-85420] is Ready metadata: creationTimestamp: "2026-07-06T20:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:10Z" name: error-404-isvc-85420.18bfce5258cb57a6 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: def48d8e-f42f-43e3-82e2-33f775e4cee1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:10.396957Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-858c3-predictor-6b7b999659-kmlhq to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceebfbab913c namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: b294cb18-a49c-4b59-a909-6e0ed2ed1475 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-858c3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec0bbbc2c3 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: f4a4b0ea-57a6-4bb6-955d-4d073d59b81d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec392f6418 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 5ceaa190-bb8e-4d8d-871c-6a947f7238bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec3b1729fa namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 9fc8f8ef-0fef-4d24-9ef3-9249254ea1d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec3e200a95 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: ce88558f-5ab4-4ac6-885f-277e681bfe17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec3e90a81b namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: d744c284-ebad-4426-82e8-1f778d4fd410 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec3e99da94 namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: ce951f00-b7f6-4418-8c17-e2bfb7186f9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec41a4a2ef namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 307509f3-2798-449c-88a3-abaac501586f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:11Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec4215db13 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 3b79f098-0199-47c7-86b5-ea181a193940 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T20:58:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:48Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfceec817cd545 namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: 7dd5e98c-d87c-4b8c-be93-9c9f87b5b5ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfcf6d32463afb namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: e56a6154-f13f-4f34-b1a2-ab58f5ad7523 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfcf6d3246a748 namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 057f442e-5769-4e82-82c1-f75ab02dbc7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T21:07:29Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-06T21:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:29Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfcf6e319ba8f0 namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 206d36d2-e2f0-4938-ae78-e50c240d525a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 290079df-dd51-4baa-8a85-08d49d7fcf25 kind: Event lastTimestamp: "2026-07-06T21:07:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: i/o timeout' metadata: creationTimestamp: "2026-07-06T21:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:29Z" name: error-404-isvc-858c3-predictor-6b7b999659-kmlhq.18bfcf6e31ec6fa8 namespace: kserve-ci-e2e-test resourceVersion: "25765" uid: 542dea6f-088f-45a5-a3bf-d72d60316f38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-858c3-predictor-6b7b999659 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 2eefc78a-51ed-4530-9765-d28a4e1df984 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'Created pod: error-404-isvc-858c3-predictor-6b7b999659-kmlhq' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-858c3-predictor-6b7b999659.18bfceebfb0775b8 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 2f4ce74a-16d3-4e99-abb7-2c45b6591479 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: f70e489d-0cfc-45fa-aaf9-612207ed5c32 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Scaled up replica set error-404-isvc-858c3-predictor-6b7b999659 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-858c3-predictor.18bfceebf96f6b2d namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: 43be22e7-ad64-4147-b141-63b41b691d84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 0c1d7dee-1c6a-4db4-bc66-78324a39d88e kind: Event lastTimestamp: "2026-07-06T20:58:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:55Z" name: error-404-isvc-858c3-predictor.18bfceef77ee86e4 namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: 25d44a96-3acc-4113-b1c4-264bf0965e67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 0c1d7dee-1c6a-4db4-bc66-78324a39d88e kind: Event lastTimestamp: "2026-07-06T20:58:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:55Z" name: error-404-isvc-858c3-predictor.18bfceef785a6175 namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 779ec322-298f-4722-9677-619f33ed9f81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 0c1d7dee-1c6a-4db4-bc66-78324a39d88e kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: error-404-isvc-858c3-predictor.18bfcef9f5589eab namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 7aa51a71-0ede-47b3-afa8-ecf8ffb924bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 0c1d7dee-1c6a-4db4-bc66-78324a39d88e kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: error-404-isvc-858c3-predictor.18bfcef9f5c56ccc namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 22783d66-3732-4074-b25c-9df479b6ca5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 69986341-6882-4903-9286-4789c5f0d9c7 kind: Event lastTimestamp: "2026-07-06T20:58:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:58Z" name: error-404-isvc-858c3.18bfceebf97fc9bc namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 3ad066bc-b227-43b9-8b9e-acebd8f9214d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 69986341-6882-4903-9286-4789c5f0d9c7 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-858c3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-858c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-858c3.18bfceebfa053a34 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 1a216c88-d11c-416f-88c6-0d4944960764 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 69986341-6882-4903-9286-4789c5f0d9c7 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-858c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:10Z" name: error-404-isvc-858c3.18bfceebfa05757b namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: c6439d65-f081-478b-be21-0810fbdf1028 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 69986341-6882-4903-9286-4789c5f0d9c7 kind: Event lastTimestamp: "2026-07-06T20:58:58Z" message: InferenceService [error-404-isvc-858c3] is Ready metadata: creationTimestamp: "2026-07-06T20:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:58Z" name: error-404-isvc-858c3.18bfcef73ecb6f2d namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: b9b908b2-8ba9-4673-9183-d4688321a464 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:47:35.160685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-87d4d-predictor-779b4f5579-8cp6c to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce5814a3352c namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 05daa974-344c-4ba4-8b4b-9a73651471bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce582ec1647e namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 2333a005-0a07-4ba0-a45b-648a2ef6ae99 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce583070bfd2 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: 09d28122-1f5e-462b-bc0d-ab4915107f99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce583308d5af namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: 94eb9d65-5fde-4d9f-ba1f-1ddccbe63cf4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce58336b988c namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 8fd8b7da-6984-43b6-af66-7658e7c1e1f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce58337436a0 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: bcdf6d09-1e2e-444b-93ba-76f6343d17f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce58364aee63 namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: be36d04d-dbc7-4d80-aa9d-c43e4e10aa98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce5836afdb81 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 2d8174c8-05fb-4a87-9a3e-40224c9efcac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:56:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:52Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfce58708447f4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 6ef35f8e-60c8-4ec7-8325-c6cc1296368a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfced94b097157 namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 79e94974-cf73-418f-93cd-3a26dcb8f8ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:50Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfced94b0a966a namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 1ad1646c-241c-40b5-924c-49727f3cb207 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 40e7d7b2-3e68-46e5-ab5f-80aed349e5f3 kind: Event lastTimestamp: "2026-07-06T20:56:52Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:52Z" name: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c.18bfced9e4f13801 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: fe263a9a-5559-4b67-b88b-766a3f14d2c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-87d4d-predictor-779b4f5579 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: d19fcec3-f42c-4dfc-adae-561b4f1865d3 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Created pod: error-404-isvc-87d4d-predictor-779b4f5579-8cp6c' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor-779b4f5579.18bfce5813f8f4ef namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 6a1734d8-507b-4b45-b234-41b77b57a7ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: f85cd18f-b9a5-4d47-9b66-1429499e6e02 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Scaled up replica set error-404-isvc-87d4d-predictor-779b4f5579 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d-predictor.18bfce58123b7183 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: e9aa1f70-394a-4c3a-8c21-e227b405f963 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:47:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 9394a9c4-0d5a-41fe-ad86-ff76b65e7818 kind: Event lastTimestamp: "2026-07-06T20:48:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:20Z" name: error-404-isvc-87d4d-predictor.18bfce5b90aa9dc1 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: e8a41c15-8dae-43d6-bbda-a33fd707f5a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:47:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 9394a9c4-0d5a-41fe-ad86-ff76b65e7818 kind: Event lastTimestamp: "2026-07-06T20:48:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:20Z" name: error-404-isvc-87d4d-predictor.18bfce5b9123d840 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 02b87072-a941-4c24-b3b6-95575d3fefdf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 9394a9c4-0d5a-41fe-ad86-ff76b65e7818 kind: Event lastTimestamp: "2026-07-06T20:48:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:50Z" name: error-404-isvc-87d4d-predictor.18bfce660df4af49 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 20e3ffc3-9056-41e1-9b82-08c4b6c39054 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 9394a9c4-0d5a-41fe-ad86-ff76b65e7818 kind: Event lastTimestamp: "2026-07-06T20:48:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:50Z" name: error-404-isvc-87d4d-predictor.18bfce660e79f533 namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 6d300369-84cb-4bff-ac69-74566805db0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 6cf19ea0-7969-4bdc-81a7-20b36e17c115 kind: Event lastTimestamp: "2026-07-06T20:48:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:22Z" name: error-404-isvc-87d4d.18bfce581239e802 namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 4d53958d-2387-4054-87d8-a253c3f37513 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 6cf19ea0-7969-4bdc-81a7-20b36e17c115 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-87d4d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-87d4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d.18bfce5812990d01 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 7ec98fa3-0d7c-474a-8e03-804af4327e34 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 6cf19ea0-7969-4bdc-81a7-20b36e17c115 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-87d4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:35Z" name: error-404-isvc-87d4d.18bfce5812993d03 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 1dc72f20-f961-4f89-af76-96038dd91c21 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 6cf19ea0-7969-4bdc-81a7-20b36e17c115 kind: Event lastTimestamp: "2026-07-06T20:48:22Z" message: InferenceService [error-404-isvc-87d4d] is Ready metadata: creationTimestamp: "2026-07-06T20:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:22Z" name: error-404-isvc-87d4d.18bfce632d503249 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: beaabb60-3bac-4b72-a8e4-ade2fa327772 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:40.209669Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef2eca576ae namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 6dade86b-99c9-4eeb-a1d6-f1942a2e8c0b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef3073ce6e1 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 7990cddd-a3ed-4d54-9906-1533e2081a8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef3094d02b1 namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: ab869192-2dc1-477a-8165-efeaa7deeb66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef30bdd551a namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: ab69306a-5ee3-4293-bbb6-a73ed1607329 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef30ca70359 namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: f9d779a7-9ad7-46a9-b020-88b6d7206439 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef30cb17999 namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: 562677d2-50ba-4f84-992f-bada752433b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef30fa7ca5e namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: 9042e567-c83e-4b1c-aa6c-44480dde319e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef310145a1a namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 1afacf1b-bc17-43c7-8276-8af42df2353f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:58:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T21:07:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:58Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcef386b68528 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: e19bafdc-a937-4ed5-a893-2b6ce4ef198b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcf7424fcddce namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: c80829a9-e42f-4f56-a0fb-2fd6b5d64981 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcf7424fe04bc namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: e61f6c1d-c61a-4360-b817-37c812d240dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: fc66437a-9311-48cd-963f-ac035c17b4a3 kind: Event lastTimestamp: "2026-07-06T21:07:58Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:58Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv.18bfcf74fb512ace namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: c52381d1-bf2b-4c64-beec-cf9f7e9b41a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-92a61-predictor-5b8cf99c8c namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 377743f5-6258-4db5-a8bf-3bd1583fe288 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Created pod: error-404-isvc-92a61-predictor-5b8cf99c8c-75dxv' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor-5b8cf99c8c.18bfcef2ec1ae66b namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: b3c863a6-664b-45b0-a59c-24e23a9042c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 23af1cc9-c6c1-40d7-9d5a-2d18713ddaff kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Scaled up replica set error-404-isvc-92a61-predictor-5b8cf99c8c from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61-predictor.18bfcef2e9da13a2 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: aff55588-8f0b-4629-8288-61baa30ae704 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 8aacff65-8b17-45c4-8b3e-2a9edc0a8b5a kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:40Z" name: error-404-isvc-92a61-predictor.18bfcef66861ba77 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: bcaa94d6-91ff-4a06-983c-3828b5ea278a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 8aacff65-8b17-45c4-8b3e-2a9edc0a8b5a kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:40Z" name: error-404-isvc-92a61-predictor.18bfcef668dc2306 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: ca4714b3-f208-4abf-85f0-aa2a7132662d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 8aacff65-8b17-45c4-8b3e-2a9edc0a8b5a kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: error-404-isvc-92a61-predictor.18bfcef9e7a47927 namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: af68b4a6-275a-4e1f-aca8-cf5a14f40b0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 8aacff65-8b17-45c4-8b3e-2a9edc0a8b5a kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: error-404-isvc-92a61-predictor.18bfcef9e7fa0236 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 5a47bdfc-aed7-4e79-a506-7ee420697f38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: a88ef38d-e8ab-44db-8a8f-9d9f0aedf36d kind: Event lastTimestamp: "2026-07-06T20:59:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:28Z" name: error-404-isvc-92a61.18bfcef2e9df789e namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: 89710f11-e5a5-402e-9e35-1e5f7bea52fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: a88ef38d-e8ab-44db-8a8f-9d9f0aedf36d kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-92a61": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-92a61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61.18bfcef2eac9508a namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 5ebe080b-408c-4db9-bdb0-b7ca1ef7531f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: a88ef38d-e8ab-44db-8a8f-9d9f0aedf36d kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-92a61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:40Z" name: error-404-isvc-92a61.18bfcef2eac99b4e namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: c3c2d300-eb84-4c62-822e-b65310df437a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: a88ef38d-e8ab-44db-8a8f-9d9f0aedf36d kind: Event lastTimestamp: "2026-07-06T20:59:28Z" message: InferenceService [error-404-isvc-92a61] is Ready metadata: creationTimestamp: "2026-07-06T20:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:28Z" name: error-404-isvc-92a61.18bfcefe4580db1e namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 3170fc77-d82c-43dc-b3a3-0b5ea5c15b55 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:07:25.241241Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-999e1-predictor-fd6f8745f-njpnw to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d2af76a06 namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: c7b247b1-de83-4b21-903b-dec21efd9cde reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d44fbca19 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: f37f9223-6270-4c1d-94c9-50c76051efaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d46c25a0c namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: 1a3584ee-42b7-48ff-b203-87b878f05963 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d49a754fa namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 2ab73186-3a64-488b-a5f0-75c99c65a4e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d4a17c593 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: f3cea7f6-038c-4901-8d15-c1b6ca0a1bbf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d4a22b2b3 namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 09fb91dc-2289-4bbc-a24e-904593b7bd58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d4d2b6553 namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: 45a4e250-b78f-4263-8462-b46c68c3267d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d4d99042a namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 18a62dc7-8c87-44a8-9a67-8d12a64f5b94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:08:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:03Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf6d9f883d4a namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 0ac8d2df-fd62-487e-8da8-a66b37545ca0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf7d7ce56524 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: b9589416-2235-49bf-b07b-ca8541dfd425 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf7d7ce8bc76 namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: 787e5c3a-3309-4b52-b598-188372a0ec0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 46b44663-269b-4ead-bbd4-09f1909c4e9b kind: Event lastTimestamp: "2026-07-06T21:08:38Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:38Z" name: error-404-isvc-999e1-predictor-fd6f8745f-njpnw.18bfcf7e27d2f6db namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: c2ae0843-e0a5-47be-bae3-f0a16e4fa46d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-999e1-predictor-fd6f8745f namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: a3ddfef6-a828-4bc0-9648-ea03d9f96d82 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Created pod: error-404-isvc-999e1-predictor-fd6f8745f-njpnw' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor-fd6f8745f.18bfcf6d2a2bd4dc namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: b4c3e193-5ceb-43bc-9685-0a395d8eda0f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: a856d94b-cf14-44a7-b4cd-3821b10c4475 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Scaled up replica set error-404-isvc-999e1-predictor-fd6f8745f from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1-predictor.18bfcf6d277088f4 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: c575a85b-d923-484d-b27c-5fc7cdf90add reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:07:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 146f3318-13f5-43b7-95c2-0c759a927bb8 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-999e1-predictor.18bfcf70a5f5cd41 namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 96d79396-5423-4b80-8826-5e27a97ac22a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:07:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 146f3318-13f5-43b7-95c2-0c759a927bb8 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:55Z" name: error-404-isvc-999e1-predictor.18bfcf70a64e776d namespace: kserve-ci-e2e-test resourceVersion: "26072" uid: 83666170-7bc9-4cf8-8732-c056c08fd1ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:08:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: 146f3318-13f5-43b7-95c2-0c759a927bb8 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: error-404-isvc-999e1-predictor.18bfcf77a3f24587 namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 40540e4d-2053-4187-952e-0ca303762b6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:08:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: 146f3318-13f5-43b7-95c2-0c759a927bb8 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: error-404-isvc-999e1-predictor.18bfcf77a4671386 namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: d1046c02-ea1c-4a07-9d15-7dc9718b47e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: 84e89f60-1641-4963-a70c-9339ee6ae619 kind: Event lastTimestamp: "2026-07-06T21:08:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:13Z" name: error-404-isvc-999e1.18bfcf6d27a510ac namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: a89242f5-605e-480f-b3eb-be235ad81ff6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: 84e89f60-1641-4963-a70c-9339ee6ae619 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-999e1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-999e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1.18bfcf6d287e2848 namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: 97b60b6d-560e-428a-85b5-ed7491dceefc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: 84e89f60-1641-4963-a70c-9339ee6ae619 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-999e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:25Z" name: error-404-isvc-999e1.18bfcf6d287e72ff namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: d458817e-96ff-471f-8ebe-c0a1e545025a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 84e89f60-1641-4963-a70c-9339ee6ae619 kind: Event lastTimestamp: "2026-07-06T21:08:13Z" message: InferenceService [error-404-isvc-999e1] is Ready metadata: creationTimestamp: "2026-07-06T21:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:13Z" name: error-404-isvc-999e1.18bfcf785c80c70e namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: 81dbb520-614d-40f1-9ff6-a84a3a3cd324 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:08:35.576229Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d7086-predictor-84cb8df759-4cmm2 to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7d8b422d20 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: b6f6f9dc-3f57-4f96-9ffa-5dd3721120d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d7086-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7d9bdd0561 namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: ccfa7af4-baf5-40c0-91fc-4cab7ec3071f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dc90eeefc namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: c82a5249-b6ff-4447-865a-91c3edcb701e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dcb3b107f namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 0e4d6046-39ca-46c5-846d-e3656108855d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dcdecf51e namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 148333ee-d41c-445b-be75-8d7eec80c3a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dce5b461b namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: 64e86655-283e-4fee-85f1-27675e0c4921 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dce64ff43 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: af353199-aec1-4ac0-9d74-494191108380 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dd29cf6cc namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: fb8e926b-7519-4085-993a-56eeae467e21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7dd3316a9a namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: 71aed4da-deba-42e4-878d-95d7f96c1ab4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:09:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:09:14Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcf7e3e0a6123 namespace: kserve-ci-e2e-test resourceVersion: "27014" uid: d9b6896f-d016-4d59-8165-0216e42f1ca2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:17:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:50Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcffeb9f9a902 namespace: kserve-ci-e2e-test resourceVersion: "30718" uid: 18a2ed43-01ab-4a86-b1da-ddc00940a588 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2 namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: ed5be343-7cc7-4eca-8436-1bf8c5dda709 kind: Event lastTimestamp: "2026-07-06T21:17:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:50Z" name: error-404-isvc-d7086-predictor-84cb8df759-4cmm2.18bfcffeb9fa3524 namespace: kserve-ci-e2e-test resourceVersion: "30719" uid: 344694ad-9847-4755-a305-4baa90ed544e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d7086-predictor-84cb8df759 namespace: kserve-ci-e2e-test resourceVersion: "26581" uid: 95412a02-3486-450d-94fa-fa79cabf93bc kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'Created pod: error-404-isvc-d7086-predictor-84cb8df759-4cmm2' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-d7086-predictor-84cb8df759.18bfcf7d89d87573 namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: 60f71e35-40fd-457b-92f6-79543d47b680 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: d7bdf4fd-6cc2-4a67-842b-ad43a1fefdfe kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Scaled up replica set error-404-isvc-d7086-predictor-84cb8df759 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-d7086-predictor.18bfcf7d87940e78 namespace: kserve-ci-e2e-test resourceVersion: "26582" uid: 8f714bc0-6ca6-4058-bbfb-e5f06245e8ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26580" uid: 87ca8bd1-5bce-4580-9aab-abd29595a08f kind: Event lastTimestamp: "2026-07-06T21:09:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:20Z" name: error-404-isvc-d7086-predictor.18bfcf81062846f4 namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: bd2fd8fa-5afd-47a3-85cd-0048309ca718 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26580" uid: 87ca8bd1-5bce-4580-9aab-abd29595a08f kind: Event lastTimestamp: "2026-07-06T21:09:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:20Z" name: error-404-isvc-d7086-predictor.18bfcf8106aff0d8 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: 94ef2152-cd0b-4e46-bfdb-1e5e7deff8d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26777" uid: 87ca8bd1-5bce-4580-9aab-abd29595a08f kind: Event lastTimestamp: "2026-07-06T21:10:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:10:05Z" name: error-404-isvc-d7086-predictor.18bfcf8b82e48227 namespace: kserve-ci-e2e-test resourceVersion: "27442" uid: 9e00c06e-0446-490a-ae7f-fc42d64a6d15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26777" uid: 87ca8bd1-5bce-4580-9aab-abd29595a08f kind: Event lastTimestamp: "2026-07-06T21:10:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:10:05Z" name: error-404-isvc-d7086-predictor.18bfcf8b8363b2bb namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: d8e70b5b-2fc8-4e0d-b45b-e36d4dc7bec6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 09b20acd-cc21-4bfc-8792-eb58f701a42e kind: Event lastTimestamp: "2026-07-06T21:09:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:09:24Z" name: error-404-isvc-d7086.18bfcf7d87ac072e namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: 8d5cbac2-9c26-4e78-a104-d04b277345b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 09b20acd-cc21-4bfc-8792-eb58f701a42e kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-d7086": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d7086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-d7086.18bfcf7d883c2770 namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: bee5bba1-03c7-4b68-8bc1-fc4d24b71859 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 09b20acd-cc21-4bfc-8792-eb58f701a42e kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d7086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:35Z" name: error-404-isvc-d7086.18bfcf7d883c5d55 namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: c6bf32e5-aed6-4fe9-9d09-b44666bb477c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: 09b20acd-cc21-4bfc-8792-eb58f701a42e kind: Event lastTimestamp: "2026-07-06T21:09:24Z" message: InferenceService [error-404-isvc-d7086] is Ready metadata: creationTimestamp: "2026-07-06T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:09:24Z" name: error-404-isvc-d7086.18bfcf88fb400f8e namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 02631e79-0804-4b82-937f-7a80fa1c286f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:15.021460Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dfad7-predictor-9f88c844f-hddzq to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:15Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce615c866423 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 6e3f8dd5-8611-4099-8cf1-4ea889ddbc0e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-dfad7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce616531ac63 namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 772629ae-bdec-4480-a4f1-550a328a41aa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce6199bb6796 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 47462500-d207-40b2-b20f-c06f36d90f41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce619bc0c06b namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: a41c438c-40a5-4273-820d-a4c8f68acb32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce619eb15efa namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: ba4d4c06-04a4-42ce-8a49-a4ecaa8b4205 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce619f7d76dd namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 498f93f9-a237-4f4e-b990-ff71d13716f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce619f87ef65 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 34b58f33-36ad-4a47-aab7-2d98b187615a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce61a24d64a1 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 0d5cd9e0-da6a-4bee-8a9a-a6a2fce1f0b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce61a2b9726e namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: da5e95a6-9922-42d7-ace5-32032d287853 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:48:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:53Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfce620825b1be namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: dd823be3-e978-412b-8e18-377ea2cfec16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:29Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfcee282276772 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 51847338-06e2-44cc-97de-610facf269e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: ce90d61a-992b-4ab1-81e4-daeed0701f53 kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:29Z" name: error-404-isvc-dfad7-predictor-9f88c844f-hddzq.18bfcee28227ebda namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 8ac5b6ee-0467-4e16-805c-dc6ed233a53b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dfad7-predictor-9f88c844f namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: e2da39ad-f0a0-4944-90df-1e3c736bce63 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: 'Created pod: error-404-isvc-dfad7-predictor-9f88c844f-hddzq' metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:15Z" name: error-404-isvc-dfad7-predictor-9f88c844f.18bfce615c05a417 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 42aa8e48-90e4-45ce-a94c-a1ebc63bcd25 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f8fc9637-2e2e-413e-b2d8-1321f9f54628 kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Scaled up replica set error-404-isvc-dfad7-predictor-9f88c844f from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:14Z" name: error-404-isvc-dfad7-predictor.18bfce615a291399 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 5f775883-9890-4786-9d59-6aa817e28bc9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 7005b109-465b-478d-b32b-0e5bace0a2fa kind: Event lastTimestamp: "2026-07-06T20:48:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:45Z" name: error-404-isvc-dfad7-predictor.18bfce64d88556fe namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: 61c42c26-b73a-4af7-b27f-e0b16a9a20b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 7005b109-465b-478d-b32b-0e5bace0a2fa kind: Event lastTimestamp: "2026-07-06T20:48:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:45Z" name: error-404-isvc-dfad7-predictor.18bfce64d8fe4587 namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 4fdb375d-6df0-4915-b304-6bebb0cb9f84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 7005b109-465b-478d-b32b-0e5bace0a2fa kind: Event lastTimestamp: "2026-07-06T20:49:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:30Z" name: error-404-isvc-dfad7-predictor.18bfce6bd6b235e8 namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: 8ebb80fc-d7ef-4949-aca4-98f4a65db174 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 7005b109-465b-478d-b32b-0e5bace0a2fa kind: Event lastTimestamp: "2026-07-06T20:49:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:30Z" name: error-404-isvc-dfad7-predictor.18bfce6bd735323a namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: a3b7edcf-fa89-4e27-b9bc-6ccb6f35c5c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 76acdc02-0acf-4aee-bf77-2d4a86a0ac81 kind: Event lastTimestamp: "2026-07-06T20:49:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:04Z" name: error-404-isvc-dfad7.18bfce615a31ce7e namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: a8585c5f-d553-4daf-9f0b-57f7924ddd0c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 76acdc02-0acf-4aee-bf77-2d4a86a0ac81 kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-dfad7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dfad7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:14Z" name: error-404-isvc-dfad7.18bfce615ade0f3e namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: 751c74fb-649a-4dcb-8768-6bc855f4768b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 76acdc02-0acf-4aee-bf77-2d4a86a0ac81 kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dfad7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:15Z" name: error-404-isvc-dfad7.18bfce615ade4d49 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: c5eecbea-a96d-47a5-9c0e-63395edd5605 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 76acdc02-0acf-4aee-bf77-2d4a86a0ac81 kind: Event lastTimestamp: "2026-07-06T20:49:04Z" message: InferenceService [error-404-isvc-dfad7] is Ready metadata: creationTimestamp: "2026-07-06T20:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:04Z" name: error-404-isvc-dfad7.18bfce6cc4780ebc namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 3b32a4bf-6d6c-4c5c-8a15-3de6a729e00d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:04.913365Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:04Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce43117a67ad namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: bf118d20-b38a-42d0-9b6c-125c9f9f63de reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce432bc914eb namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 5a1adda0-3a3d-4ad3-983f-ffe1721a7d3f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce432d967c1d namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 10d94463-05f4-4f2d-9a9f-c45fb3c4c556 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4330219b83 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 5f1f0ede-7b1f-4e08-92e5-302880f73f42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4330a7aa2f namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 43fb88bf-6271-4409-a52b-6ef87ebec935 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:09Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" metadata: creationTimestamp: "2026-07-06T20:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:09Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce44280caa76 namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: ee3baa79-f348-4d73-8382-614554217623 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:32Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" in 22.976s (22.976s including waiting). Image size: 1555746730 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:32Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce49818fba6d namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 32ad9467-2438-4e77-a0a1-51482694fe13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:32Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4986cb5252 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 81e0d226-6428-4d74-aaf7-61f4d34f49e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:32Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce49883a752f namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 6f9cce6c-4cc9-47cf-8de0-8a05a9a12d2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:32Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T20:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:32Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce49884548ff namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: ffc1896e-0569-41d8-90d0-c68dee1e5dae reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.968s (2.968s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:35Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4a3931a487 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: 2cd127d9-1dd6-41c7-a166-dbbff92695cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:35Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4a3fd9ff24 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: e1c6180d-d110-49d3-b821-2c57622ad3e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:46:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:35Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4a40f92ffa namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 90905f8e-724d-40f8-ac32-e8cd1695f14e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-06T20:46:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:47:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:32Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce4a7baa86ab namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 694780ec-748d-4c11-9e21-20701004c52e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:14Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce6155baa61a namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 9b85197d-149c-4e6e-9bd9-532e23866ba5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:14Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce6155bb23fa namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 0a9c0c7d-7fdc-4e28-b779-a2c864be03bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: add345a8-fd5e-4fed-8711-a9da088c8e3d kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5.18bfce61fff5d458 namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 4a0082bf-b110-4da7-a96f-af41ea919d29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7fbf85bc5d namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 4e4ccdfc-971d-4053-805f-848ecae1feae kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7fbf85bc5d-kp6d5' metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:04Z" name: isvc-sklearn-graph-1-predictor-7fbf85bc5d.18bfce431105621e namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: dd5989b0-22a8-43c5-ad1c-0129a8dc1407 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: 9d2f799d-15b6-4398-9dd9-248da59085dd kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7fbf85bc5d from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:04Z" name: isvc-sklearn-graph-1-predictor.18bfce430df10889 namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 8edfdc13-01e1-4a4e-98ad-bb54a4a224ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: 2abc210a-4793-4d3c-ab40-b79ba0ae2a2a kind: Event lastTimestamp: "2026-07-06T20:46:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:49Z" name: isvc-sklearn-graph-1-predictor.18bfce468c6efec6 namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 9360b092-7acf-4ccc-a10b-5eaff81e083d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: 2abc210a-4793-4d3c-ab40-b79ba0ae2a2a kind: Event lastTimestamp: "2026-07-06T20:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:49Z" name: isvc-sklearn-graph-1-predictor.18bfce468cd5747c namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 9ba81570-3a88-43d6-8f94-a19aebf585f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: 2abc210a-4793-4d3c-ab40-b79ba0ae2a2a kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:05Z" name: isvc-sklearn-graph-1-predictor.18bfce510ab8d0c5 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: bc9532d0-8ae6-4539-9e9c-43df6f6a98f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: 2abc210a-4793-4d3c-ab40-b79ba0ae2a2a kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:05Z" name: isvc-sklearn-graph-1-predictor.18bfce510b34cad3 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 930d9b72-9c39-49a0-b033-6feeb4d139d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 99c15806-0244-42c9-a695-d5d8f2c8a48f kind: Event lastTimestamp: "2026-07-06T20:47:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:42Z" name: isvc-sklearn-graph-1.18bfce430e039f06 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 576192d0-8fe2-46ed-afa7-d2c2c7fb53dc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 99c15806-0244-42c9-a695-d5d8f2c8a48f kind: Event lastTimestamp: "2026-07-06T20:47:42Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-06T20:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:42Z" name: isvc-sklearn-graph-1.18bfce59e1166a61 namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 1a994085-8334-46eb-9913-9467a430a396 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:05.297559Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce432860d08a namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 404defd4-1109-469e-863c-957d2c8994a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce4337353629 namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: 89845259-fa59-423e-9651-cf8a5a9cb26a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:06Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce436575ed73 namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: bf9d352b-d47c-4cec-8586-32d75caedf70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:06Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce4366f6f90a namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: c164ab9b-fd8e-4aa9-8b51-d98928b5ce1d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:09Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" in 3.071s (3.071s including waiting). Image size: 300633537 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:09Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce441e123ded namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: bc7bc476-4a76-4e43-92c8-f5b10de561f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:09Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce44222374a8 namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 184efc42-1297-4c41-ac9e-bf2f830a978c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:46:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:09Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce4422b13842 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: a62a2e34-329a-480c-992d-2cca3d78009b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:13Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" metadata: creationTimestamp: "2026-07-06T20:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:13Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce452ae0f816 namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 9dcd8541-a233-476a-991a-df67e32ae79a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" in 12.746s (12.746s including waiting). Image size: 1555746730 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce48229c78e0 namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: d9365927-d04a-409e-8619-8c7c6a0cab16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce482697eb67 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 8829b001-5c7d-43fe-9513-42883fe3675a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce482706f2d0 namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: a7334201-733b-4caf-b64f-b4a34042bb43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce482712c92a namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 953b1ead-8e9c-4a64-b120-5546dd32bff1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.228s (2.228s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:28Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce48abed21ca namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: 81b6f460-06bf-49fa-a3c4-f92e7a65478d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:29Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce48aecf77ae namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 5ed61ce1-22af-412d-9be2-4f8082f588c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:46:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:29Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce48af3c00d8 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 9457ffd0-de3d-4367-b47c-ace22fc4e0c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:46:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce48e912fd0a namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 9a9f7d25-2a5b-415f-978c-e439ab0e4b97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:14Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce6151040f1a namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: c714eac9-2d59-4999-b3fb-7cc8f9f27a7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:14Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce61510474f4 namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 728902d0-2414-4952-acfb-d35ce7bf238e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 0e0c7517-edbb-448f-9869-9e7df46d3e8f kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz.18bfce6197503e5f namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 5f78b990-674d-469f-a802-975e63aa604a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6664c8d89d namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 37f87d59-ba1a-4467-be66-0ecdef9357da kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6664c8d89d-hd2nz' metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-2-predictor-6664c8d89d.18bfce4327e8188b namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 91204c08-7bae-4639-8f68-0992bd41633a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 1f120bc2-85c2-4bbb-978c-453094c8281d kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6664c8d89d from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:05Z" name: isvc-sklearn-graph-2-predictor.18bfce4322223b7d namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 947170a9-c8aa-436b-b599-61e41448e02c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 1e321276-8edd-4c1b-af3c-9bac5edc8f27 kind: Event lastTimestamp: "2026-07-06T20:47:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:20Z" name: isvc-sklearn-graph-2-predictor.18bfce46a0aecc9c namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: fc9adb75-e16a-4cad-a531-fa7633f37b66 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 1e321276-8edd-4c1b-af3c-9bac5edc8f27 kind: Event lastTimestamp: "2026-07-06T20:47:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:20Z" name: isvc-sklearn-graph-2-predictor.18bfce46a126f90c namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: a16387ad-28de-454b-a8fd-83fa45cd0b96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 1e321276-8edd-4c1b-af3c-9bac5edc8f27 kind: Event lastTimestamp: "2026-07-06T20:48:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:05Z" name: isvc-sklearn-graph-2-predictor.18bfce581ad75aae namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: d61aaf5c-e79d-40e3-95e4-6af76627c6d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 1e321276-8edd-4c1b-af3c-9bac5edc8f27 kind: Event lastTimestamp: "2026-07-06T20:48:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:05Z" name: isvc-sklearn-graph-2-predictor.18bfce581b5855e7 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 26a086a2-9149-49f3-89b5-8ed6a413b85a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: 2e17c001-91a7-46ca-929a-478f1d2ef18a kind: Event lastTimestamp: "2026-07-06T20:47:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:36Z" name: isvc-sklearn-graph-2.18bfce4322394db7 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: faf5b267-8474-4747-9fea-024abc87820b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 2e17c001-91a7-46ca-929a-478f1d2ef18a kind: Event lastTimestamp: "2026-07-06T20:47:36Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-06T20:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:36Z" name: isvc-sklearn-graph-2.18bfce584deb1fbd namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: c5b1da14-77c5-4a91-bca3-7629eb14902d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:05.137601Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-blq59 to ip-10-0-137-106.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce431ed80864 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: f091a80c-c035-420b-acd5-c701a0129fb0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce43275e8d0f namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 81c1a917-e1de-4219-8ba2-b5656f625845 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce435c2de9b9 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: fad53fdc-1bc4-496d-8980-513d7f4ccfce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce435de2960c namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 25e24f07-f7da-41ef-b854-d79e3cb975f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce4361194b2f namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: b167f473-00c1-4984-98a0-b5da24aae550 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce436183107f namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: da72d406-0461-4115-b7ce-54f2adaed6b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-06T20:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce4464546ceb namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 59538c02-37ec-42cd-aefe-102b5a6176f8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:32Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.386s (22.386s including waiting). Image size: 1305293053 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce499aad8f87 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 236c89a6-4abd-4d3b-9eaf-366fb26cd3db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce499fe6fa00 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 68beb11c-f6b9-4055-9867-88a9419b1d5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce49a0994362 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 28aad06c-60c0-411e-8e43-fae383299594 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T20:46:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce49a0a98aeb namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 7f0c35c1-636e-4001-ac12-04af1bf6fc16 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.567s (2.567s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce4a39b12853 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 2176de64-03df-4d32-aed6-422a0d54f899 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce4a403906b2 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: b581d109-1aa2-4d9c-b171-435d50c3b1c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:46:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce4a41537f97 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: 60cb2179-f658-410d-9556-e5986495c417 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:47:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce4ab783930e namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: d5fffa2e-68f7-4cac-930c-c3b57e5dab1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce61603ba66c namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: c98a5961-7254-497d-b3d3-beed35f9dccb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-blq59 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 80c0447d-61f7-43fe-bb81-ca0a00c65f22 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-blq59.18bfce61603c0dd7 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 655e710f-065b-4bea-92f0-85ab922bdeb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-106.ec2.internal source: component: kubelet host: ip-10-0-137-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: ad307eb2-013e-4b12-a618-e272f8995979 kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-blq59' metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:05Z" name: isvc-xgboost-graph-predictor-555fc69d56.18bfce431e552f4e namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 4360dc29-baa1-4bdc-95e7-5faf24fccd84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: baeec402-d50e-48ae-8075-4083ace77d5d kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:05Z" name: isvc-xgboost-graph-predictor.18bfce431b1443d0 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 99ea12c6-7817-47c5-9bc1-0d99e7e6dd46 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 7ce9cc65-6639-4102-948f-f2f69fdfd945 kind: Event lastTimestamp: "2026-07-06T20:47:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:05Z" name: isvc-xgboost-graph-predictor.18bfce4699a2f729 namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 844461b8-b0da-444c-a1be-f5a1395575b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 7ce9cc65-6639-4102-948f-f2f69fdfd945 kind: Event lastTimestamp: "2026-07-06T20:47:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:05Z" name: isvc-xgboost-graph-predictor.18bfce469a19b5bb namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 1fc95446-8ca3-4a59-bc9c-5a9305f12977 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 7ce9cc65-6639-4102-948f-f2f69fdfd945 kind: Event lastTimestamp: "2026-07-06T20:47:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:50Z" name: isvc-xgboost-graph-predictor.18bfce54950653c7 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 98828aac-ac33-4f3f-a0e3-18bdecc3e080 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:47:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 7ce9cc65-6639-4102-948f-f2f69fdfd945 kind: Event lastTimestamp: "2026-07-06T20:47:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:50Z" name: isvc-xgboost-graph-predictor.18bfce54957968f1 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 5b24e462-c76d-4d0c-82d4-dfd3d8974efb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: 52cb1f86-a371-4e03-b675-45661f90c30e kind: Event lastTimestamp: "2026-07-06T20:47:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:33Z" name: isvc-xgboost-graph.18bfce431b39701a namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 09fb3053-dc30-459b-823f-190686695970 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 52cb1f86-a371-4e03-b675-45661f90c30e kind: Event lastTimestamp: "2026-07-06T20:47:33Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-06T20:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:33Z" name: isvc-xgboost-graph.18bfce57c813e15c namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: db6f17c5-3d39-4527-b29d-003baa7c4639 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:04.594729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7dc7f99b6d-qtdht to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:04Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce5eef0b223a namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 32eadbf0-0c3e-4cf2-a7a1-cb206fa77556 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: "2026-07-06T20:48:05Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:05Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce5f087958b2 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 667df945-13a5-47ad-aa28-77b6066f95dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: "2026-07-06T20:48:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:05Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce5f09c8f8cb namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: c94e11d1-cc34-4fb6-b986-ec5fc118071c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: "2026-07-06T20:48:05Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-06T20:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:05Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce5f0cbdf457 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: e1189659-7924-4852-b4c0-3b6d5e74f41c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: "2026-07-06T20:48:05Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-06T20:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:05Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce5f0d26d01d namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: d6002e58-4b04-4b0b-a399-c7aec7262cf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:14Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce614872268b namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 0dc82ff9-234e-477b-b16e-3e9c0cf32d1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7dc7f99b6d-qtdht namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 440f131a-4c58-44d7-8fd7-f3279ce6d988 kind: Event lastTimestamp: "2026-07-06T20:48:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:41Z" name: model-chainer-7dc7f99b6d-qtdht.18bfce61ad0b8e88 namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 80f90500-e018-4244-8bee-187bf1cff647 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7dc7f99b6d namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: c0084cc6-c75f-4a5a-8c2c-3e59594a993a kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: 'Created pod: model-chainer-7dc7f99b6d-qtdht' metadata: creationTimestamp: "2026-07-06T20:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:04Z" name: model-chainer-7dc7f99b6d.18bfce5eee8fad02 namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 549f8aab-1262-4ee0-bcf3-5017ac12cfda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 00ebceb2-2c9e-40b1-9971-2bd95b123b64 kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: Scaled up replica set model-chainer-7dc7f99b6d from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:04Z" name: model-chainer.18bfce5eed6a8699 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 5bbc508a-200b-4359-a3df-498fad89cbc3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 08ade248-d40c-4e9b-87d2-a2b6dfda0118 kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-06T20:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:04Z" name: model-chainer.18bfce5ef3fc9b4e namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: fc988f1a-6886-45dd-85f2-1e388cfb143c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:29.711389Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-24ddf-599f97958c-svd5v to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:29Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef07ae66198 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 3b7526ef-7f64-4110-bd60-230249be1746 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: "2026-07-06T20:58:30Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:30Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef093ea165a namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 8f3642a2-c834-4f70-997e-86a94eb5538f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24ddf} kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: "2026-07-06T20:58:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:30Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef09546ae8a namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 3d9cb39c-ee79-40b8-ad5e-cbc9a1a56bcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24ddf} kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: "2026-07-06T20:58:30Z" message: 'Created container: sequence-graph-24ddf' metadata: creationTimestamp: "2026-07-06T20:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:30Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef097fe9c8b namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: f2ca8c06-d90e-4f98-86e7-d12ac2e5c794 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24ddf} kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: "2026-07-06T20:58:30Z" message: Started container sequence-graph-24ddf metadata: creationTimestamp: "2026-07-06T20:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:30Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef09866d7db namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: 9fa82af5-a0fd-4b71-8c98-b0df878119d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24ddf} kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: "2026-07-06T20:58:39Z" message: Stopping container sequence-graph-24ddf metadata: creationTimestamp: "2026-07-06T20:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:39Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef2d4f34541 namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: c1609270-6fec-4455-b787-ff5b989f0816 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24ddf} kind: Pod name: sequence-graph-24ddf-599f97958c-svd5v namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: cd0b809d-fc36-47f5-b35a-1c5098831d8e kind: Event lastTimestamp: "2026-07-06T20:59:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:06Z" name: sequence-graph-24ddf-599f97958c-svd5v.18bfcef3420bc2ef namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 0852c76d-4c5e-4149-861e-d3fe95d6dd82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-24ddf-599f97958c namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 2f2914a4-ac8c-4266-86e8-ef0e6e818d7e kind: Event lastTimestamp: "2026-07-06T20:58:29Z" message: 'Created pod: sequence-graph-24ddf-599f97958c-svd5v' metadata: creationTimestamp: "2026-07-06T20:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:29Z" name: sequence-graph-24ddf-599f97958c.18bfcef07a35efbf namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: a2d7dbd7-e392-43c5-a0ac-f1773abd04b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-24ddf namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: 07615cea-df93-4753-af9d-2e2b9aaf5d18 kind: Event lastTimestamp: "2026-07-06T20:58:29Z" message: Scaled up replica set sequence-graph-24ddf-599f97958c from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:29Z" name: sequence-graph-24ddf.18bfcef0797cc2f4 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: b2025284-2e2e-4715-a600-be632072e873 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-24ddf namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: e955001b-6c96-45ff-a062-b7163370b620 kind: Event lastTimestamp: "2026-07-06T20:58:29Z" message: InferenceGraph [sequence-graph-24ddf] is Ready metadata: creationTimestamp: "2026-07-06T20:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:29Z" name: sequence-graph-24ddf.18bfcef081be73a4 namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: ab72223f-f53e-4f3b-b946-261a23e63599 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:59:40.058651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-92a61-78458fdc5-fngpc to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf00dbec6731 namespace: kserve-ci-e2e-test resourceVersion: "22066" uid: 8912b8c2-ce2c-4cfc-91d1-0b6c9656e8f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf00f58cca32 namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: ab08c4ae-5b25-404e-8b85-f9a2ef295e18 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92a61} kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf00f7336feb namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: ed5d3d40-7bfe-47cb-b2fb-9e80def5980c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92a61} kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: 'Created container: sequence-graph-92a61' metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf00fa6a4108 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 6730df16-8b7c-4c34-aa59-94c7eb1c90b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92a61} kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: Started container sequence-graph-92a61 metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf00fad53517 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 849b7e27-f740-43c8-a8fe-7eb78a18dce7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92a61} kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: "2026-07-06T21:07:54Z" message: Stopping container sequence-graph-92a61 metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:54Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf7409598a3c namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: bd3ca961-8e3d-4196-9a8c-7e521e78b96c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-92a61} kind: Pod name: sequence-graph-92a61-78458fdc5-fngpc namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 3d985656-6014-4421-86e4-dcf731ef19e3 kind: Event lastTimestamp: "2026-07-06T21:08:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:21Z" name: sequence-graph-92a61-78458fdc5-fngpc.18bfcf7489d23a73 namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: d408c900-80fe-4264-afb9-9d1e9a0afd86 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-92a61-78458fdc5 namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d3658e2d-adab-4c30-ab58-5f22f71179bb kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: 'Created pod: sequence-graph-92a61-78458fdc5-fngpc' metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61-78458fdc5.18bfcf00db6af15a namespace: kserve-ci-e2e-test resourceVersion: "22062" uid: a780b55f-9dd2-4012-a79e-f54a6bef638a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-92a61 namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: d4dbc252-eca3-459b-87c7-b3eea68f20e7 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: Scaled up replica set sequence-graph-92a61-78458fdc5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61.18bfcf00da7dea76 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: c9b78e3c-9d24-4d8f-8193-7bd70b321fc0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-92a61 namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: 9b5d2a66-2d30-4338-b117-5b0b696357f1 kind: Event lastTimestamp: "2026-07-06T20:59:40Z" message: InferenceGraph [sequence-graph-92a61] is Ready metadata: creationTimestamp: "2026-07-06T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:40Z" name: sequence-graph-92a61.18bfcf00e20ca575 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: b3180332-f03b-4a79-96c6-d2603157b7bd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:59:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-92a61 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: a8729ae9-ea28-4102-a64b-7ef3735e1b0c kind: Event lastTimestamp: "2026-07-06T21:00:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:00:25Z" name: sequence-graph-92a61.18bfcf04596745a3 namespace: kserve-ci-e2e-test resourceVersion: "22422" uid: a34e01a1-ff56-4e55-836e-618bbc6edd4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:59:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-92a61 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: a8729ae9-ea28-4102-a64b-7ef3735e1b0c kind: Event lastTimestamp: "2026-07-06T21:00:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:00:25Z" name: sequence-graph-92a61.18bfcf0459dfdd6c namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 4372b2de-7a8d-4501-aeb4-f7f65c3a158e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:49:14.879046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-dfad7-5f5d97c8c5-9l9jn to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:49:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:49:14Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfce6f4c50b9f6 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 50a4ea5c-8777-4a12-9344-a57968644b97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: "2026-07-06T20:49:15Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:49:15Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfce6f656da695 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 6b54484f-56aa-4daf-90a0-6376feb9d3eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dfad7} kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: "2026-07-06T20:49:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:15Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfce6f66b41311 namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 139c3108-ffaf-4537-be17-24cf254b83a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dfad7} kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: "2026-07-06T20:49:15Z" message: 'Created container: sequence-graph-dfad7' metadata: creationTimestamp: "2026-07-06T20:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:15Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfce6f69311d1e namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: ec7b1062-ccb6-4eb4-ab4b-e1cd580fe574 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dfad7} kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: "2026-07-06T20:49:15Z" message: Started container sequence-graph-dfad7 metadata: creationTimestamp: "2026-07-06T20:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:15Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfce6f699a036d namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 99bdaa00-2af7-4ddc-8d4c-c9165d4e6da5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dfad7} kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Stopping container sequence-graph-dfad7 metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:29Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfcee27835df84 namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 02308984-38ed-458c-970b-5554d947441c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dfad7} kind: Pod name: sequence-graph-dfad7-5f5d97c8c5-9l9jn namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: a72cc3a0-b146-4295-add1-c67bdead913a kind: Event lastTimestamp: "2026-07-06T20:57:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:56Z" name: sequence-graph-dfad7-5f5d97c8c5-9l9jn.18bfcee2f4da198d namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: ec1f3d78-8f2a-4097-af1b-5c2808ccb32f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-dfad7-5f5d97c8c5 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: ea0b0321-7b07-4450-8b6b-f3b7e41af280 kind: Event lastTimestamp: "2026-07-06T20:49:14Z" message: 'Created pod: sequence-graph-dfad7-5f5d97c8c5-9l9jn' metadata: creationTimestamp: "2026-07-06T20:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:14Z" name: sequence-graph-dfad7-5f5d97c8c5.18bfce6f4bd5c314 namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: 0fbb55ed-2507-4325-b5b5-257b35d28720 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: c54fa922-c39e-4a68-a923-6513963ba3fc kind: Event lastTimestamp: "2026-07-06T20:49:14Z" message: Scaled up replica set sequence-graph-dfad7-5f5d97c8c5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:14Z" name: sequence-graph-dfad7.18bfce6f4aae165c namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: 965938cb-87bd-41a8-a9f9-a0710f54e4be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: 252609e5-2ad8-457b-8080-e17b06cdb873 kind: Event lastTimestamp: "2026-07-06T20:49:14Z" message: InferenceGraph [sequence-graph-dfad7] is Ready metadata: creationTimestamp: "2026-07-06T20:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:14Z" name: sequence-graph-dfad7.18bfce6f4f26d8a6 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: ddea8f49-11b8-4fc1-8dff-bd3958709023 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 252609e5-2ad8-457b-8080-e17b06cdb873 kind: Event lastTimestamp: "2026-07-06T20:49:14Z" message: 'Failed to update status for InferenceGraph "sequence-graph-dfad7": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-dfad7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:14Z" name: sequence-graph-dfad7.18bfce6f527d534b namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: fc5b73e8-9e5b-4cc9-9b8a-9e8d5806e23d reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 252609e5-2ad8-457b-8080-e17b06cdb873 kind: Event lastTimestamp: "2026-07-06T20:49:14Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-dfad7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:14Z" name: sequence-graph-dfad7.18bfce6f527d99d5 namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: 45a320bd-9b0a-4d84-8044-db5f9bc23c54 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: b24d2476-1966-44ee-9892-3b524147c5e6 kind: Event lastTimestamp: "2026-07-06T20:49:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:44Z" name: sequence-graph-dfad7.18bfce72c9a93866 namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: e737dada-96fa-47c8-b254-9f32692e9afb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: b24d2476-1966-44ee-9892-3b524147c5e6 kind: Event lastTimestamp: "2026-07-06T20:49:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:44Z" name: sequence-graph-dfad7.18bfce72ca0283fb namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 8213417d-8f81-4b1b-bf11-03a67c266df7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:08:25.136611Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-999e1-5df54fbb4c-8s8kp to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:08:25Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7b1d0249b4 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: b9fddffc-d9fb-4a10-a3bc-b6d3d47d62f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:25Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-999e1-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:25Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7b282c6ec8 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: b77183f2-8f7f-414e-802b-285daf2d8b9b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:08:26Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:08:26Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7b59fe3d4c namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 2ea973f0-0a2f-41c6-b390-3e0af408fe76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-999e1} kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:08:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T21:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:26Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7b5b60f449 namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 8c0483cb-df92-41be-b43b-532c180bb483 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-999e1} kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:08:26Z" message: 'Created container: splitter-graph-999e1' metadata: creationTimestamp: "2026-07-06T21:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:26Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7b5ecc5878 namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: a1a37237-2397-4103-84ea-3e266bf4f4b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-999e1} kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:08:26Z" message: Started container splitter-graph-999e1 metadata: creationTimestamp: "2026-07-06T21:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:26Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7b5f3969ac namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: f0f6c644-d19a-4674-8d4f-9e6599c45256 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-999e1} kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Stopping container splitter-graph-999e1 metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7d76b7d868 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: ccb06524-6394-49cd-a552-316bd0cde9e4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-999e1} kind: Pod name: splitter-graph-999e1-5df54fbb4c-8s8kp namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 4db11150-7f15-404d-a5a8-d63156df21a1 kind: Event lastTimestamp: "2026-07-06T21:09:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:09:02Z" name: splitter-graph-999e1-5df54fbb4c-8s8kp.18bfcf7e0d6bf9b0 namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 2fece8fb-4f74-4e64-98a4-b4a9742fc842 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-999e1-5df54fbb4c namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 7e279c46-a15d-4ae0-8ab4-a87fbcf7df13 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'Created pod: splitter-graph-999e1-5df54fbb4c-8s8kp' metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: splitter-graph-999e1-5df54fbb4c.18bfcf7b1c7886af namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 4b13f889-7bab-42c2-8e80-276163223a4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-999e1 namespace: kserve-ci-e2e-test resourceVersion: "26372" uid: 80ffc969-77bd-46c3-ab93-f00b4338ca34 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: Scaled up replica set splitter-graph-999e1-5df54fbb4c from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: splitter-graph-999e1.18bfcf7b1b7758de namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: d478b7b5-bd90-4e0b-9985-4cb41d3ec6c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-999e1 namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 11d9ca4d-7a41-481a-8611-807d0b6573a8 kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: InferenceGraph [splitter-graph-999e1] is Ready metadata: creationTimestamp: "2026-07-06T21:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:25Z" name: splitter-graph-999e1.18bfcf7b20ac2b22 namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: d1998e19-7928-444e-b146-5a8363ef91b1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:09:35.469087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27193" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-d7086-ff98df69b-m7n8z to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcf8b7d26a9e0 namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: c49b7486-931e-4d69-a87e-780a95a83658 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcf8b964773b5 namespace: kserve-ci-e2e-test resourceVersion: "27216" uid: e6104593-5768-41c1-a95a-0a317b514342 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d7086} kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcf8b97c18838 namespace: kserve-ci-e2e-test resourceVersion: "27218" uid: caefbe18-9d28-4d55-ad58-b728290a87ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d7086} kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: 'Created container: splitter-graph-d7086' metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcf8b9a94082e namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 71b36d34-fa7f-43f2-a885-afb043f21f1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d7086} kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: Started container splitter-graph-d7086 metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcf8b9b00476b namespace: kserve-ci-e2e-test resourceVersion: "27220" uid: 01689943-3e52-4107-ac74-dff9964f8a86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d7086} kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: "2026-07-06T21:17:50Z" message: Stopping container splitter-graph-d7086 metadata: creationTimestamp: "2026-07-06T21:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:50Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcffeac3c33d0 namespace: kserve-ci-e2e-test resourceVersion: "30685" uid: 28f45aca-ab8a-436f-bcde-a3926d5ff6e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:17:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d7086} kind: Pod name: splitter-graph-d7086-ff98df69b-m7n8z namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: 79f616e2-6b86-41d4-8f5d-24205f04fcef kind: Event lastTimestamp: "2026-07-06T21:18:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:18:17Z" name: splitter-graph-d7086-ff98df69b-m7n8z.18bfcfff190b3d7f namespace: kserve-ci-e2e-test resourceVersion: "30916" uid: a6440f63-42c8-48ea-8351-4b94fbb8b6e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-d7086-ff98df69b namespace: kserve-ci-e2e-test resourceVersion: "27187" uid: ab469f1d-f77f-4498-950f-c9d990683981 kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: 'Created pod: splitter-graph-d7086-ff98df69b-m7n8z' metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086-ff98df69b.18bfcf8b7c871e9a namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: dec43ded-3960-49db-8688-58d47b8b25b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-d7086 namespace: kserve-ci-e2e-test resourceVersion: "27185" uid: 7a72c9a5-68c4-4761-a0de-6be0c30782f1 kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: Scaled up replica set splitter-graph-d7086-ff98df69b from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086.18bfcf8b7b60aadc namespace: kserve-ci-e2e-test resourceVersion: "27189" uid: 22dd5e96-6f08-4448-a2fb-fc86c4c855f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-d7086 namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: dc3c2b56-50f3-44e2-a394-fc9a40558fbf kind: Event lastTimestamp: "2026-07-06T21:09:35Z" message: InferenceGraph [splitter-graph-d7086] is Ready metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:09:35Z" name: splitter-graph-d7086.18bfcf8b806e2c4c namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 0b9bb83d-facc-4a8b-835b-122667f5c0a5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:09:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-d7086 namespace: kserve-ci-e2e-test resourceVersion: "27186" uid: 12286359-caeb-4822-b67f-8f896b37c588 kind: Event lastTimestamp: "2026-07-06T21:10:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:10:05Z" name: splitter-graph-d7086.18bfcf8efa5591ee namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 3622af39-ec0d-4688-8dc3-f83dff6932eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:09:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-d7086 namespace: kserve-ci-e2e-test resourceVersion: "27186" uid: 12286359-caeb-4822-b67f-8f896b37c588 kind: Event lastTimestamp: "2026-07-06T21:10:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:10:05Z" name: splitter-graph-d7086.18bfcf8efaddba09 namespace: kserve-ci-e2e-test resourceVersion: "27441" uid: 1b917313-7357-433b-9c86-9ddea8d826bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:57:29.827192Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-24ddf-predictor-b8f9d9964-25s57 to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:57:29Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee289860dfd namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: b90f0d30-0007-4587-95cd-02fc117e9d6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2a3295c75 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 826b1423-aff6-497e-8cbc-30e8e62c2f14 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2a6b73b34 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 1b633af9-2838-46da-9c96-00ef55fef92d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2a95c9d9d namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 06496e51-73d8-435e-a9b0-877732740639 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2a9cd6a0a namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 57a5b0f7-b634-41b0-8a84-398ed84a8f7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2a9d73777 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 6dd6a4cb-d7a3-452a-a200-b3c9e49c3da9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2ae752148 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 2f4d4651-0898-4e24-b2cf-6a72d9851dbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:57:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:30Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee2aee67994 namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 8e807685-7dff-4c22-a24e-83dc6a902b58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:58:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:08Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcee3251e9bc3 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: aa54aca7-3973-4152-bf4d-e63fda4f723c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcef2e7cde3e4 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: fd7f444a-3484-4983-be75-27baea777d82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcef2e7ce6522 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 6ca2c3cd-6f99-4e1b-a8e2-4be078a3f776 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: acc97a0d-30b2-4875-b3c6-931994dfca80 kind: Event lastTimestamp: "2026-07-06T20:58:43Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:43Z" name: success-200-isvc-24ddf-predictor-b8f9d9964-25s57.18bfcef3ad5501f5 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 851afcc1-1654-448d-85dc-5919cf12fd9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-24ddf-predictor-b8f9d9964 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: de02ad80-8ea9-40ac-bd5b-e6e0e84f55f9 kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: 'Created pod: success-200-isvc-24ddf-predictor-b8f9d9964-25s57' metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:29Z" name: success-200-isvc-24ddf-predictor-b8f9d9964.18bfcee288d69c75 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 03cf5b7f-9566-46c1-8cec-626e56a76da2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: 72dd041c-0c2d-4545-bfa9-03e653b3d2f1 kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Scaled up replica set success-200-isvc-24ddf-predictor-b8f9d9964 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:29Z" name: success-200-isvc-24ddf-predictor.18bfcee28314ea85 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 661b5d3b-354b-4d9e-a2f4-61fadfd733f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: f991bc6c-d0e4-4a26-9626-0ca4de7458c6 kind: Event lastTimestamp: "2026-07-06T20:57:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:59Z" name: success-200-isvc-24ddf-predictor.18bfcee6018d3699 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 5523cbed-3f86-4cca-bcf5-a3992983d95c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: f991bc6c-d0e4-4a26-9626-0ca4de7458c6 kind: Event lastTimestamp: "2026-07-06T20:57:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:59Z" name: success-200-isvc-24ddf-predictor.18bfcee60205b316 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 58b6ba25-4cdc-4782-9fb1-d3c90eddf293 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: f991bc6c-d0e4-4a26-9626-0ca4de7458c6 kind: Event lastTimestamp: "2026-07-06T20:58:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:29Z" name: success-200-isvc-24ddf-predictor.18bfceed00033eae namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 4e65373a-b840-4132-8c7e-5218f70eca19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24ddf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: f991bc6c-d0e4-4a26-9626-0ca4de7458c6 kind: Event lastTimestamp: "2026-07-06T20:58:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:29Z" name: success-200-isvc-24ddf-predictor.18bfceed0053a84c namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: ad858dc2-39ec-40b6-8dfb-104fa5c8c8b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-24ddf namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: 8e47daa3-fd96-4ffe-bfa8-1a55825fbef4 kind: Event lastTimestamp: "2026-07-06T20:58:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:18Z" name: success-200-isvc-24ddf.18bfcee283516ab5 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 5aec6c0b-89d6-4459-8321-df56f2e2f6a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-24ddf namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: 8e47daa3-fd96-4ffe-bfa8-1a55825fbef4 kind: Event lastTimestamp: "2026-07-06T20:58:18Z" message: InferenceService [success-200-isvc-24ddf] is Ready metadata: creationTimestamp: "2026-07-06T20:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:18Z" name: success-200-isvc-24ddf.18bfceede5209a4f namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: cf0238c7-dec8-46dc-9adb-d3f290af7293 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:56:49.983003Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-305f4-predictor-c7fb94cc6-66jqf to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:56:49Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced942a00190 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: edd6fbda-2ee3-4c0f-b193-bbe01dc141c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-305f4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:50Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced94ecb3876 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: adabcfab-96f7-4391-9335-5cd4ae86f449 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced980f75ab5 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: c46d7c0f-437f-4d95-b184-2a31aeaaa7c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced982e4b33d namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: a7d280cd-03b8-4a4f-93a8-a9b9e2e9eeb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced98623baa5 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: b87bea46-836e-4cd7-a880-8451f11344b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced986aa9552 namespace: kserve-ci-e2e-test resourceVersion: "20070" uid: 3186b543-d2e9-4625-837e-0ca8c69ff4d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced986b48a8b namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 4557ed34-200a-4260-a40d-b217ad34533b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced989d86923 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: ed12152e-142c-41f6-ab9b-a67c002d53d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced98a4f527f namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 165457e0-6522-47b5-b2b6-b53dbe37bbda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:57:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:37Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfced98c681dc2 namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 9756f0f5-2bf8-457e-9d51-877b174138ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfceebf3224412 namespace: kserve-ci-e2e-test resourceVersion: "20969" uid: f655fd50-0218-4f53-982f-2ed6b850c0ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfceebf323aa22 namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 0ad5492e-2d0b-446f-abdb-4236bf012aeb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: d4b9da3e-0e8f-4fd4-9550-82b899e84d89 kind: Event lastTimestamp: "2026-07-06T20:58:12Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:12Z" name: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf.18bfceec68fae7bf namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: a5a42c3f-1447-4a77-b3f2-aa8cdce25db0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-305f4-predictor-c7fb94cc6 namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 9d1fe316-23dc-4ddb-a83e-b16e32937831 kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: 'Created pod: success-200-isvc-305f4-predictor-c7fb94cc6-66jqf' metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:49Z" name: success-200-isvc-305f4-predictor-c7fb94cc6.18bfced941f199fc namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 51b1a3c4-6610-415b-98ba-37a209a7a13a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 37043c2f-7eea-4b0a-99d4-0a3bb46583ff kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: Scaled up replica set success-200-isvc-305f4-predictor-c7fb94cc6 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:49Z" name: success-200-isvc-305f4-predictor.18bfced93e40f733 namespace: kserve-ci-e2e-test resourceVersion: "19970" uid: 01a55253-377a-4260-aa30-30fc519552a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 1e194bf3-4150-4b98-b1c8-c6de350fec5c kind: Event lastTimestamp: "2026-07-06T20:57:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:34Z" name: success-200-isvc-305f4-predictor.18bfcedcbcdef91e namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: d0b31db2-06e3-4a1a-9837-7fd3cbbe4edb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 1e194bf3-4150-4b98-b1c8-c6de350fec5c kind: Event lastTimestamp: "2026-07-06T20:57:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:34Z" name: success-200-isvc-305f4-predictor.18bfcedcbd5360f6 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 2b12c45b-103f-41e3-b283-e662d946df84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:57:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 1e194bf3-4150-4b98-b1c8-c6de350fec5c kind: Event lastTimestamp: "2026-07-06T20:58:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:04Z" name: success-200-isvc-305f4-predictor.18bfcee03c1a3316 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 481b0f92-ba6b-4b1e-aa88-0e045d06f112 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:57:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-305f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 1e194bf3-4150-4b98-b1c8-c6de350fec5c kind: Event lastTimestamp: "2026-07-06T20:58:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:05Z" name: success-200-isvc-305f4-predictor.18bfcee03c98b91d namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: d2d9d01a-96a9-4ac3-b65f-794650135943 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 4b7512c6-79cc-4737-bb63-85e3f2ae7e4b kind: Event lastTimestamp: "2026-07-06T20:57:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:57:47Z" name: success-200-isvc-305f4.18bfced93e4a1c83 namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: e2d5dee8-416a-4f1e-b842-1aaf88599b14 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 4b7512c6-79cc-4737-bb63-85e3f2ae7e4b kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-305f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-305f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:49Z" name: success-200-isvc-305f4.18bfced93edb8e8f namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 2e547896-713c-409a-99a7-da7223b58521 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 4b7512c6-79cc-4737-bb63-85e3f2ae7e4b kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-305f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:49Z" name: success-200-isvc-305f4.18bfced93edbc752 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: e1ab8359-a5f2-4c95-bb98-274aba7303eb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-305f4 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: 4b7512c6-79cc-4737-bb63-85e3f2ae7e4b kind: Event lastTimestamp: "2026-07-06T20:57:47Z" message: InferenceService [success-200-isvc-305f4] is Ready metadata: creationTimestamp: "2026-07-06T20:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:57:47Z" name: success-200-isvc-305f4.18bfcee69fbe785a namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: e593ef5b-a3fa-4687-98b5-1041c4e9a1e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:07:54.972226Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:07:54Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf74171247a0 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 035f7a78-8f96-4738-a11f-5c3b2381bf15 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf7430aded76 namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: d4c8f1f1-1916-4a27-bc2d-3d67e6597902 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf7432821281 namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: 71fa379f-1809-410a-bcaa-02cd1ad3ac16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf74353c941b namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 89ac88f2-c0f2-4045-bd1d-7fa2e17f6304 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf7435af566f namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 737d59fb-5abf-40c0-a3e5-2e3aca9f354b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf7435ba6d3e namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: b4fd5e0e-0951-4867-a9f1-b8ca5c5b4177 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf74397f1fd8 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 52224bff-b482-450c-8349-7652f63eeedd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf743a02954f namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 539ce589-7b50-49b2-8c46-5d969b084b9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:07:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:08:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:33Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfcf74a50a603b namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: 006f2d9c-e453-4b9d-9817-e34b32697447 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:25:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:25:14Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfd06611f7abdc namespace: kserve-ci-e2e-test resourceVersion: "33783" uid: 38304517-2343-4ff5-80e3-c9058bfa1ebc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 65e3ebfc-8b2e-4020-b04c-6f294c74ac4e kind: Event lastTimestamp: "2026-07-06T21:25:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:25:14Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2.18bfd06611f839a8 namespace: kserve-ci-e2e-test resourceVersion: "33784" uid: e701a4cb-acc6-4325-bf42-8be83c74af40 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-32e32-predictor-5fb6b9c6c8 namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: 00ff613e-f008-4dcf-b792-0c2544a015ac kind: Event lastTimestamp: "2026-07-06T21:07:54Z" message: 'Created pod: success-200-isvc-32e32-predictor-5fb6b9c6c8-2hsv2' metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:54Z" name: success-200-isvc-32e32-predictor-5fb6b9c6c8.18bfcf7416801072 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: e762375f-9f8b-4ad9-bcde-5aabe7624f3f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "25986" uid: 0198f309-803c-4a5a-93c4-f99459de8632 kind: Event lastTimestamp: "2026-07-06T21:07:54Z" message: Scaled up replica set success-200-isvc-32e32-predictor-5fb6b9c6c8 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:54Z" name: success-200-isvc-32e32-predictor.18bfcf7413d6b33c namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: cb92cbea-1f67-425e-b648-9ac15dbbc766 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: 9b1bd42c-aea3-48b8-9027-437539a7e05a kind: Event lastTimestamp: "2026-07-06T21:08:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:55Z" name: success-200-isvc-32e32-predictor.18bfcf7792bc0493 namespace: kserve-ci-e2e-test resourceVersion: "26843" uid: 11bfcb45-dd86-4d14-86c1-431ed88e485f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: 9b1bd42c-aea3-48b8-9027-437539a7e05a kind: Event lastTimestamp: "2026-07-06T21:08:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:55Z" name: success-200-isvc-32e32-predictor.18bfcf779314b9cd namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: a13b9fee-f69e-40ed-94fb-04a6243e9b0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: 9b1bd42c-aea3-48b8-9027-437539a7e05a kind: Event lastTimestamp: "2026-07-06T21:08:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:24Z" name: success-200-isvc-32e32-predictor.18bfcf7b124b9d56 namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 1b2bcf5c-92bb-4019-aaae-1f5911572930 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e32-predictor namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: 9b1bd42c-aea3-48b8-9027-437539a7e05a kind: Event lastTimestamp: "2026-07-06T21:08:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:24Z" name: success-200-isvc-32e32-predictor.18bfcf7b12c9fee1 namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 52d6f986-9483-4949-9a4b-1b230dc45b8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: a9808c9f-7f9e-491d-800f-4d2536f479c6 kind: Event lastTimestamp: "2026-07-06T21:08:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:43Z" name: success-200-isvc-32e32.18bfcf741476b025 namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: 3691de76-8536-46d6-a2d0-65bc1961f728 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: a9808c9f-7f9e-491d-800f-4d2536f479c6 kind: Event lastTimestamp: "2026-07-06T21:07:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-32e32": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-32e32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:54Z" name: success-200-isvc-32e32.18bfcf7414de7330 namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: fee6ec2c-eabb-4f1e-b7d9-f24426b6ed06 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: a9808c9f-7f9e-491d-800f-4d2536f479c6 kind: Event lastTimestamp: "2026-07-06T21:07:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-32e32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:54Z" name: success-200-isvc-32e32.18bfcf7414dece76 namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 2387dba3-7468-48a6-8a13-739138431b7b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: a9808c9f-7f9e-491d-800f-4d2536f479c6 kind: Event lastTimestamp: "2026-07-06T21:08:43Z" message: InferenceService [success-200-isvc-32e32] is Ready metadata: creationTimestamp: "2026-07-06T21:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:43Z" name: success-200-isvc-32e32.18bfcf7f6a8d83a2 namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: 97ef3dda-904e-468a-9795-f3d77d59f072 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:04.612237Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-85420-predictor-8c694c98b-cnn9k to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:04Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce42ff878b7c namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: bb668950-5039-47d9-af5d-1483e478a047 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-85420-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:04Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce4308e97273 namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 04fee4ca-f172-462f-b315-05476f539a17 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:05Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce433ccb6366 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: bc24c217-051f-4eb5-bc51-2c0352147f48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:05Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" metadata: creationTimestamp: "2026-07-06T20:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:05Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce433ec1eedc namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 908b28e8-d092-4954-bee6-dfdc9ea5b6cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" in 15.661s (15.662s including waiting). Image size: 1335633000 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce46e449790e namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 5e7a4174-7073-4d72-a4fd-aa63f1ac3945 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce46e8720fac namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 870ab0a9-0e09-4930-a521-2c29c8ada01a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce46e8e49254 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: b5ce1ded-7952-49a9-ab90-ac3acdcabceb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:21Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T20:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:21Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce46e8f16af5 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 11b1de8a-9a34-4ed2-914b-a2536b7084b2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.548s (2.548s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:23Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce4780d79757 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 49d1d041-013f-40fe-b08b-11a2ddca59f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:24Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce4784fa222e namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 4ab91f26-e4a2-455a-b888-bda7716933c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:46:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:24Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce47856d080d namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 44227b09-0800-4187-9b48-cd2a7dc11d11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:47:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:00Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce4798913a08 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: 0abced01-6ac2-4e02-9f48-b114ce9741f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce5809d8dc90 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 43af06f4-ab53-419c-8ef5-7a213acfa9ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce5809d99bd6 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 73cc15a1-b393-4d09-a396-13bc75af02a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-85420-predictor-8c694c98b-cnn9k namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 601c9d0f-a9df-4b87-942a-173b3ac9d8b2 kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-85420-predictor-8c694c98b-cnn9k.18bfce5820bd128e namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: a8ac037a-e199-40ea-9bb9-4874148f0901 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-85420-predictor-8c694c98b namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: e231572b-7e8e-4ee1-8403-6dad076b670c kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: 'Created pod: success-200-isvc-85420-predictor-8c694c98b-cnn9k' metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:04Z" name: success-200-isvc-85420-predictor-8c694c98b.18bfce42feadc600 namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 82d21fd7-7904-48e3-a7e8-0e8ec626194d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 325fb0ae-235b-49c4-96cf-bf6b811cd543 kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: Scaled up replica set success-200-isvc-85420-predictor-8c694c98b from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:04Z" name: success-200-isvc-85420-predictor.18bfce42f5f3587d namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 2d254e29-53c1-454a-b1a8-414833c3f7b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 121ba014-f406-4f1f-a844-45fe20d19d59 kind: Event lastTimestamp: "2026-07-06T20:47:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:04Z" name: success-200-isvc-85420-predictor.18bfce467ba23b58 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 63a29bc6-c074-464f-8430-543970d2b260 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 121ba014-f406-4f1f-a844-45fe20d19d59 kind: Event lastTimestamp: "2026-07-06T20:47:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:04Z" name: success-200-isvc-85420-predictor.18bfce467c190552 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 2ca9098f-dd28-4d80-97aa-3fc53a13d98e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:47:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 121ba014-f406-4f1f-a844-45fe20d19d59 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-85420-predictor.18bfce54784bfc10 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 4ffb4256-969a-4d0c-8902-80fa1daa84e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:47:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-85420-predictor namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 121ba014-f406-4f1f-a844-45fe20d19d59 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-85420-predictor.18bfce5478bc7f83 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 82cde504-c339-4f07-9c66-33203547efe5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-85420 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: aefafed1-2f17-4fee-af19-64af7f9465d5 kind: Event lastTimestamp: "2026-07-06T20:47:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:10Z" name: success-200-isvc-85420.18bfce42fcaf868f namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 2c021fc4-7402-4c1f-8192-30e6a4bbfbaa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-85420 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: aefafed1-2f17-4fee-af19-64af7f9465d5 kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-85420": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-85420": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:46:04Z" name: success-200-isvc-85420.18bfce42fd36b9be namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 4018f798-c26e-4662-87f0-3e547fdfe80a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-85420 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: aefafed1-2f17-4fee-af19-64af7f9465d5 kind: Event lastTimestamp: "2026-07-06T20:46:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-85420": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:46:04Z" name: success-200-isvc-85420.18bfce42fd36fb12 namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 00140428-8075-46a3-aadd-943f790062ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-85420 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: aefafed1-2f17-4fee-af19-64af7f9465d5 kind: Event lastTimestamp: "2026-07-06T20:47:10Z" message: InferenceService [success-200-isvc-85420] is Ready metadata: creationTimestamp: "2026-07-06T20:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:10Z" name: success-200-isvc-85420.18bfce52556a7931 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 7bc59e22-f340-449e-8b4b-d06dc1936399 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:10.280656Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceebf4bce39f namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 0f4cc686-2fff-4d24-a39b-fd2fadfb1bd4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20984" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec0e9ad3cf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 4c28bf2f-5a31-4723-8fe0-771a53e176af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec104a646f namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: c5d24621-627a-4d43-98ba-c40aa0b38336 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec13943695 namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: e6b4eccc-3cbc-4956-8afd-c001b48b500c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec13ff017c namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: e6fd8d9d-ceb5-4483-bf4d-dd588e516f7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec14087282 namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 05d55013-a419-4cd3-b6a1-d1fdfeeae10a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec16c70d7e namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 95ee09b9-9570-4f64-87e5-c43d0c57fa58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec17552827 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 510253b7-1db9-4225-a8f5-ef845389507e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T21:07:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:27Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfceec45551c0c namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 8fd9dce1-dcfd-4b84-9f21-07c189b8250f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfcf6d2527bff3 namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 0bb7e6aa-3e87-49e5-ab69-0c3cdfe3be04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfcf6d252839a6 namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 6d8c8d81-6f58-4935-8c79-2416cfee6969 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 0353baf0-e073-4f81-aa0c-68e01997550c kind: Event lastTimestamp: "2026-07-06T21:07:27Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:27Z" name: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt.18bfcf6db9f8c0d2 namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: e72be28c-0c16-4afe-a595-1c792079330a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-858c3-predictor-845dd5dbd6 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: ac310a8f-27d3-4997-925c-591578738d46 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'Created pod: success-200-isvc-858c3-predictor-845dd5dbd6-f4bmt' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor-845dd5dbd6.18bfceebf444c1df namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 56a1973a-4eea-4873-a90b-d79cdfb84a54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 3313d011-dd2f-4eb0-8481-bc02cbf962db kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: Scaled up replica set success-200-isvc-858c3-predictor-845dd5dbd6 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3-predictor.18bfceebf24e421c namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 944494bd-bdbb-452e-8aba-849075c2cf6e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 9598b437-46e5-44e8-9a34-4d2267103ce7 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-858c3-predictor.18bfceef715a7611 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: 0d139643-c254-48e7-9398-050602a043ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 9598b437-46e5-44e8-9a34-4d2267103ce7 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-858c3-predictor.18bfceef71b4fa46 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: f440cb74-8862-4dc6-b0ed-15ab93208e27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 9598b437-46e5-44e8-9a34-4d2267103ce7 kind: Event lastTimestamp: "2026-07-06T20:59:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:25Z" name: success-200-isvc-858c3-predictor.18bfcef66fab2b71 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 2fa62f36-4cd1-4b9e-93b1-aa3d4e1657f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-858c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 9598b437-46e5-44e8-9a34-4d2267103ce7 kind: Event lastTimestamp: "2026-07-06T20:59:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:25Z" name: success-200-isvc-858c3-predictor.18bfcef670274343 namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: 66abb580-da51-41e2-953a-e7a5fcf68d8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 77d11e97-6f60-4261-8cb7-e03c91b374c8 kind: Event lastTimestamp: "2026-07-06T20:58:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:57Z" name: success-200-isvc-858c3.18bfceebf26b8047 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: b48c0919-f884-4ce1-a304-a72a683b7eef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 77d11e97-6f60-4261-8cb7-e03c91b374c8 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-858c3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-858c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3.18bfceebf3021310 namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 7935ee2d-d12e-42d5-bc7e-2236b5cfe8ad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 77d11e97-6f60-4261-8cb7-e03c91b374c8 kind: Event lastTimestamp: "2026-07-06T20:58:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-858c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:10Z" name: success-200-isvc-858c3.18bfceebf302597c namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 94624e37-6f06-4de0-88ea-38b0c615e32e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-858c3 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: 77d11e97-6f60-4261-8cb7-e03c91b374c8 kind: Event lastTimestamp: "2026-07-06T20:58:57Z" message: InferenceService [success-200-isvc-858c3] is Ready metadata: creationTimestamp: "2026-07-06T20:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:57Z" name: success-200-isvc-858c3.18bfcef702462985 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: bde99ef4-eec7-4057-bafb-96c2773911ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:47:35.030376Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce580cdeedf3 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: 2c626bb5-57aa-420a-a699-6bc47925ce9f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce5826723a98 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 2f58f3cc-5b08-47bd-a615-9082878cbd0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce5828d3ded6 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 5cb57bde-cdb8-41e2-8e88-b932b86714ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce582bfdf0d0 namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: 1cf1a58e-9c6f-4d31-8305-1795211a7c0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce582c684486 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 242e2930-dd7c-418d-83a7-a55941e07e84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce582c72bb5d namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: 3d5b2fd3-f6c5-491d-921f-468100f36415 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce582fac8ea3 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 3b359b17-dc04-40c1-a9cd-a08bf154883e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce58300c8b35 namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 5f820e01-1a61-4c89-b035-b1fa0a53fe64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfce5834370307 namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: 4dfcf7dd-9d7c-4c99-8b87-cebb765628b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:49Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfced94329b1f1 namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: cc0f71fb-5902-41f8-9972-90bb5e699cba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:50Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfced94329e287 namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 1d6babac-b683-4e36-8954-254e60e1475c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ff6dd3d9-a337-45c3-a51a-218105f7ff8e kind: Event lastTimestamp: "2026-07-06T20:56:51Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:51Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2.18bfced9a90ab573 namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: f07ba7a9-7cea-4045-8344-2fa1b7a5011b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-87d4d-predictor-5dfdb8946b namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: c416a14f-0029-4bea-ad03-70b06dbf81cf kind: Event lastTimestamp: "2026-07-06T20:47:35Z" message: 'Created pod: success-200-isvc-87d4d-predictor-5dfdb8946b-9snv2' metadata: creationTimestamp: "2026-07-06T20:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:35Z" name: success-200-isvc-87d4d-predictor-5dfdb8946b.18bfce580b85908a namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 6f6aa05c-29f6-4135-8656-1e13d4ee5923 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 12f90e72-9421-415f-9400-cb1ee72f1c15 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: Scaled up replica set success-200-isvc-87d4d-predictor-5dfdb8946b from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-87d4d-predictor.18bfce58096f8278 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 44f8c143-e517-4c1f-b12a-5b6fae3866fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:47:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 367155ff-5422-4dea-a13e-bf4e5001a979 kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:04Z" name: success-200-isvc-87d4d-predictor.18bfce5b87c20805 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 5165ae96-ccc2-4bd9-a95f-71c3f6dbb8a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:47:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 367155ff-5422-4dea-a13e-bf4e5001a979 kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:05Z" name: success-200-isvc-87d4d-predictor.18bfce5b88373918 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 85807641-1695-43e0-91ef-274624f4e0db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:48:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 367155ff-5422-4dea-a13e-bf4e5001a979 kind: Event lastTimestamp: "2026-07-06T20:48:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:50Z" name: success-200-isvc-87d4d-predictor.18bfce62864e6b36 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: ba7d99af-58da-435c-a6a7-805df76a5d60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:48:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87d4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 367155ff-5422-4dea-a13e-bf4e5001a979 kind: Event lastTimestamp: "2026-07-06T20:48:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:50Z" name: success-200-isvc-87d4d-predictor.18bfce628716ebc5 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 24cfdb12-d4d5-446b-8302-56974af7667f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 7246621a-a5bc-4481-a8a0-9524264f9e87 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:21Z" name: success-200-isvc-87d4d.18bfce5809b1acd3 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 9a4fcf94-3c8f-4b31-b8d5-d509136b277d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 7246621a-a5bc-4481-a8a0-9524264f9e87 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-87d4d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87d4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-87d4d.18bfce580a5b3687 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: fdd37295-f1ec-4368-af53-02b9ae129944 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 7246621a-a5bc-4481-a8a0-9524264f9e87 kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87d4d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:34Z" name: success-200-isvc-87d4d.18bfce580a5b926d namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: df7b54e8-2c75-4e86-bbcb-9f72469bf793 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 7246621a-a5bc-4481-a8a0-9524264f9e87 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: InferenceService [success-200-isvc-87d4d] is Ready metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:21Z" name: success-200-isvc-87d4d.18bfce62f14b54c1 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: b550372b-024b-4f1d-af18-1d99b3696aaf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:40.040914Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-92a61-predictor-f78d96bf8-l44xt to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef2e29684a2 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: fbd2a438-15b3-474b-99c7-7d9e439a4235 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef2fc19d2f8 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 3722925f-bfc3-4bc1-a72f-02b47e20270e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef2fe5bd043 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 7117388e-582e-4a07-863b-1c94b0085d0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef301302150 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: cebfcf1e-fbb0-4df3-9df2-bb2c5a588434 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef3019fea96 namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 94117756-8667-4047-aeb1-02e933d3c13d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef301a9f04a namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: e49ea06d-d1d5-4e4c-ad24-ef4d6492d600 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef304c4adbd namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ed35ac9b-a2ec-4756-9e9f-30dba4a77069 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef3053863fe namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: d24113c8-6d05-4a2d-a4b3-e658a8133e3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T21:07:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:57Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcef34ad4cade namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: c4be02ff-ba4a-4f3c-a6cf-cb01415e0e47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcf741ae04443 namespace: kserve-ci-e2e-test resourceVersion: "26019" uid: 77d738ea-37e4-46c3-a1f5-da5bc868ed20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcf741ae08b7c namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: 99651388-c9f8-4d3a-93af-be487eb50d4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 6611d492-131b-4cb5-81e2-e67702aac1f2 kind: Event lastTimestamp: "2026-07-06T21:07:57Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:57Z" name: success-200-isvc-92a61-predictor-f78d96bf8-l44xt.18bfcf74bf3f41f9 namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: c7eeb5c8-a78f-4d8a-ac26-c1bcf890a2b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-92a61-predictor-f78d96bf8 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 5cddc8a1-513b-4c31-b151-d24667560e02 kind: Event lastTimestamp: "2026-07-06T20:58:40Z" message: 'Created pod: success-200-isvc-92a61-predictor-f78d96bf8-l44xt' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61-predictor-f78d96bf8.18bfcef2e20b7cba namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 55699c77-3010-4b3b-84ac-ff8c85817da1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 529c3938-b2c9-40c8-a20d-212b766b2188 kind: Event lastTimestamp: "2026-07-06T20:58:39Z" message: Scaled up replica set success-200-isvc-92a61-predictor-f78d96bf8 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:39Z" name: success-200-isvc-92a61-predictor.18bfcef2de1580ab namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 09dc6c61-5222-46d1-996d-10fa634ea867 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:58:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: c8293728-e643-4cbf-b1a0-a6fd936a11b9 kind: Event lastTimestamp: "2026-07-06T20:59:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:55Z" name: success-200-isvc-92a61-predictor.18bfcef65cd5ae3b namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 2a7239c8-4b2a-448a-abd5-ed0740f9dd79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:58:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: c8293728-e643-4cbf-b1a0-a6fd936a11b9 kind: Event lastTimestamp: "2026-07-06T20:59:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:55Z" name: success-200-isvc-92a61-predictor.18bfcef65d499d54 namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: c19c6a04-f8c8-40cd-946e-dbde29d30273 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: c8293728-e643-4cbf-b1a0-a6fd936a11b9 kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: success-200-isvc-92a61-predictor.18bfcef9dc875741 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: d024fc57-3087-4832-a8d5-00dd904b4446 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-92a61-predictor namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: c8293728-e643-4cbf-b1a0-a6fd936a11b9 kind: Event lastTimestamp: "2026-07-06T20:59:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:10Z" name: success-200-isvc-92a61-predictor.18bfcef9dd7245a6 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 3190f350-f26f-45ee-8425-2b393e4b131b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:58:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 0c6d752b-1925-436c-9408-5362a4fe815e kind: Event lastTimestamp: "2026-07-06T20:59:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:27Z" name: success-200-isvc-92a61.18bfcef2de276898 namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 00aadadf-4d98-495b-924b-c2fe94f14ac0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 0c6d752b-1925-436c-9408-5362a4fe815e kind: Event lastTimestamp: "2026-07-06T20:58:39Z" message: 'Failed to update status for InferenceService "success-200-isvc-92a61": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-92a61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:39Z" name: success-200-isvc-92a61.18bfcef2deba0a61 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: b4097fb9-7138-430a-943d-e1f4d8119c74 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 0c6d752b-1925-436c-9408-5362a4fe815e kind: Event lastTimestamp: "2026-07-06T20:58:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-92a61": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:40Z" name: success-200-isvc-92a61.18bfcef2deba4531 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 3dfd77f6-dfc0-409a-a07d-18d744b385ef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-92a61 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 0c6d752b-1925-436c-9408-5362a4fe815e kind: Event lastTimestamp: "2026-07-06T20:59:27Z" message: InferenceService [success-200-isvc-92a61] is Ready metadata: creationTimestamp: "2026-07-06T20:59:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:27Z" name: success-200-isvc-92a61.18bfcefe08bb198b namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 78913c4f-205e-4757-a54f-c59e210c2817 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:07:25.132812Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-999e1-predictor-57944785f4-q888m to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d2480f381 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: e3b5b64a-c93a-42c2-b8a5-0f46eb5f5ef6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d3ef794e0 namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: db84f441-15a3-4cea-84f8-07611ea8d767 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d409fc282 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: af6f26fd-f459-4c5a-bb88-c32394633977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d442ac61a namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: e811420a-ec6c-43ef-a34a-21b59a3bec76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d44cc8145 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 46334135-2b59-49da-a4e9-e360a09134d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d44d749ed namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 0e3e667f-13cf-492e-88d8-a8143a81b477 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d48f37d21 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: cd2003a7-f80e-422b-a816-660c20f84d50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d496db879 namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: 1bf59ead-3573-4109-be64-5a3064ed0308 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:07:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:08:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:02Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf6d63a9d3c9 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 2fae3291-b012-43ed-9d7f-95b69df27a7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf7d85236168 namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: d7f40f55-2425-4bc7-9af0-38a0c97a37ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf7d8523b43b namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 0eb0ad4c-804c-43f4-90fa-ccbae3e759d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-999e1-predictor-57944785f4-q888m namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: fae36521-e6ea-446b-892c-51cc9ec16d81 kind: Event lastTimestamp: "2026-07-06T21:08:37Z" message: 'Readiness probe failed: Get "https://10.134.0.46:8643/healthz": dial tcp 10.134.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:37Z" name: success-200-isvc-999e1-predictor-57944785f4-q888m.18bfcf7debd6ac66 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: f5945d37-05aa-4c66-903d-23708369b95e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-999e1-predictor-57944785f4 namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 7db47a6d-11b9-45d2-baa6-e1aa63202a34 kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Created pod: success-200-isvc-999e1-predictor-57944785f4-q888m' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor-57944785f4.18bfcf6d24192c02 namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 2a2aa402-bb86-4dcb-b79a-dad0b2ea68d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25612" uid: 7005b5ca-b92d-44fa-82da-932205a6cc7e kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: Scaled up replica set success-200-isvc-999e1-predictor-57944785f4 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1-predictor.18bfcf6d1f065e3f namespace: kserve-ci-e2e-test resourceVersion: "25615" uid: 0cbcdbd4-2238-4034-9603-e85ea882fd08 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:07:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: 82126b5f-f542-4f82-b69f-51b6e1f374ae kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: success-200-isvc-999e1-predictor.18bfcf709dc21267 namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: 5af72758-0141-4230-8c8e-17458f59b329 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:07:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: 82126b5f-f542-4f82-b69f-51b6e1f374ae kind: Event lastTimestamp: "2026-07-06T21:08:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:07:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:25Z" name: success-200-isvc-999e1-predictor.18bfcf709e6aea45 namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: d9d85878-b3da-4456-a907-fb57924f2fd5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: 82126b5f-f542-4f82-b69f-51b6e1f374ae kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-999e1-predictor.18bfcf741d53e79d namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: 08dd4f4b-9452-4454-bfeb-5846a3aea41b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-999e1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: 82126b5f-f542-4f82-b69f-51b6e1f374ae kind: Event lastTimestamp: "2026-07-06T21:07:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:07:55Z" name: success-200-isvc-999e1-predictor.18bfcf741e19343d namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: f428dd46-02ea-43be-8355-fdb1c4292721 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: 150e6807-0e5d-4aae-8223-d00cdde31eab kind: Event lastTimestamp: "2026-07-06T21:08:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:12Z" name: success-200-isvc-999e1.18bfcf6d1f1bf88e namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: ac462ecb-5c74-4e84-9173-c98aadf5aa2b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: 150e6807-0e5d-4aae-8223-d00cdde31eab kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-999e1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-999e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1.18bfcf6d1fa4a3b9 namespace: kserve-ci-e2e-test resourceVersion: "25618" uid: 74b56bf4-ffec-4303-b432-8567f66dca44 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: 150e6807-0e5d-4aae-8223-d00cdde31eab kind: Event lastTimestamp: "2026-07-06T21:07:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-999e1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:07:25Z" name: success-200-isvc-999e1.18bfcf6d1fa4d987 namespace: kserve-ci-e2e-test resourceVersion: "25620" uid: fc6d62c6-a76a-4fbf-8b06-f7432c2aef93 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-999e1 namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: 150e6807-0e5d-4aae-8223-d00cdde31eab kind: Event lastTimestamp: "2026-07-06T21:08:12Z" message: InferenceService [success-200-isvc-999e1] is Ready metadata: creationTimestamp: "2026-07-06T21:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:12Z" name: success-200-isvc-999e1.18bfcf7820119f73 namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 6fa8d643-27f0-47b0-bfc7-eca07229de3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:08:35.415326Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d7086-predictor-6848586898-5cvmn to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7d81ab024a namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: cff27ff1-5208-499f-86ee-221f94357d8f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d7086-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7d8fd10917 namespace: kserve-ci-e2e-test resourceVersion: "26599" uid: 38c6fe1f-daaa-42c1-9f36-e3cdf50ec4c3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dbf4adedb namespace: kserve-ci-e2e-test resourceVersion: "26619" uid: d90ed992-3e25-4b7b-81b3-013a166047a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dc1e23a19 namespace: kserve-ci-e2e-test resourceVersion: "26621" uid: ca684da8-04d7-4120-8a1b-34b1cf0f89db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dc560c3c7 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 50f1ce71-ec7c-4232-8e96-3835994b8b16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dc60bfd8c namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 1d37c859-075e-472b-88fb-b903e4139244 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dc6178e13 namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: b351b746-bcf8-4af0-9618-2c3116ca88b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dc9ce134c namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: eff6803a-b676-43dc-a939-9af15520bc81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:08:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:36Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7dca41003d namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: acd0432f-269a-4185-b5a9-55b9ce5acba3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:09:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:09:13Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcf7e01e7d6c3 namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: b11e7310-c671-4901-9046-efa79d27a632 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:17:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T21:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:50Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcffeb6ca901e namespace: kserve-ci-e2e-test resourceVersion: "30708" uid: f7677ab6-477f-43d4-ad95-aed38b08c54c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:17:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T21:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:50Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcffeb6cb01e0 namespace: kserve-ci-e2e-test resourceVersion: "30710" uid: ce2eb37d-46d1-4028-9a61-c6fded143123 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:17:54Z" message: 'Readiness probe failed: Get "https://10.134.0.50:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-06T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:54Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcfffb2456248 namespace: kserve-ci-e2e-test resourceVersion: "30754" uid: 4879ccb7-1f48-4aa1-933c-09e5f1f6af8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:17:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d7086-predictor-6848586898-5cvmn namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a34b63b4-ac86-40cf-9b74-fe0cb52f9b75 kind: Event lastTimestamp: "2026-07-06T21:17:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: i/o timeout' metadata: creationTimestamp: "2026-07-06T21:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:17:54Z" name: success-200-isvc-d7086-predictor-6848586898-5cvmn.18bfcfffb287200f namespace: kserve-ci-e2e-test resourceVersion: "30755" uid: e120b21f-5224-4282-a933-e3b18f1e3695 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d7086-predictor-6848586898 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: ef6ca05f-93f4-4b27-9e82-db168bf280df kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'Created pod: success-200-isvc-d7086-predictor-6848586898-5cvmn' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-d7086-predictor-6848586898.18bfcf7d80e80ee0 namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 2c13c0e9-e5f8-4d6b-91c5-ed2f33f56a5c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 52bfafdc-5d6b-4946-bc26-7917a84920bc kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: Scaled up replica set success-200-isvc-d7086-predictor-6848586898 from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-d7086-predictor.18bfcf7d7e9fbcc8 namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 71760972-031f-473c-88a1-e2a3cf8c35d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: e6a14899-6747-4341-9501-dae9381b9c41 kind: Event lastTimestamp: "2026-07-06T21:09:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:20Z" name: success-200-isvc-d7086-predictor.18bfcf80fd6087eb namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: d7f53252-64b5-48e0-8b4a-a857bb207f11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: e6a14899-6747-4341-9501-dae9381b9c41 kind: Event lastTimestamp: "2026-07-06T21:09:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:20Z" name: success-200-isvc-d7086-predictor.18bfcf80fdda3139 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: 02ad4f7c-d959-44a1-877e-cfb939e9e786 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: e6a14899-6747-4341-9501-dae9381b9c41 kind: Event lastTimestamp: "2026-07-06T21:10:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:10:05Z" name: success-200-isvc-d7086-predictor.18bfcf8b7b0b1c10 namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: a5bf0e23-147c-4557-8599-afe34deb03c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T21:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d7086-predictor namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: e6a14899-6747-4341-9501-dae9381b9c41 kind: Event lastTimestamp: "2026-07-06T21:10:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T21:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:10:05Z" name: success-200-isvc-d7086-predictor.18bfcf8b7ba023e1 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 5f6da37a-fbcc-4eb1-bec3-48ca89d1a8b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 774eee6d-4bcf-4c35-a910-7fa886842889 kind: Event lastTimestamp: "2026-07-06T21:09:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:09:23Z" name: success-200-isvc-d7086.18bfcf7d7ef2bbbe namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: 44a23a56-26c4-4538-8676-91ee46089ed7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 774eee6d-4bcf-4c35-a910-7fa886842889 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'Failed to update status for InferenceService "success-200-isvc-d7086": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d7086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-d7086.18bfcf7d7f768d85 namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: b45d6479-dc26-4a1e-826b-b32670228ad1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 774eee6d-4bcf-4c35-a910-7fa886842889 kind: Event lastTimestamp: "2026-07-06T21:08:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d7086": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:35Z" name: success-200-isvc-d7086.18bfcf7d7f76e826 namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: baec4772-04b6-4197-944b-035f54b8454d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:09:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d7086 namespace: kserve-ci-e2e-test resourceVersion: "27091" uid: 774eee6d-4bcf-4c35-a910-7fa886842889 kind: Event lastTimestamp: "2026-07-06T21:09:23Z" message: InferenceService [success-200-isvc-d7086] is Ready metadata: creationTimestamp: "2026-07-06T21:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:09:23Z" name: success-200-isvc-d7086.18bfcf88bec5b90d namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: 09f8bffc-3e74-46ad-8c57-769ab7782a78 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:14.904442Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq to ip-10-0-142-142.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:14Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce61558ce020 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 9d7bd066-16ed-4af0-b751-f193283b608f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce616f9a96d8 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: f9cdbb0d-9728-4326-8d10-2f6bfa439893 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce6171390193 namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: f7d6375c-b63b-4c7b-97d3-cfd122310b9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce6174a0cb6f namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 7e87a737-e297-4753-bc8b-3642d286c48b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce61751d74cc namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 54cd7086-da0e-42f0-a868-d29c3206fd3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce61752701d9 namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: c81af17c-9e4a-4306-bedb-284f5dd1e7f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce617819cab7 namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: 927eb816-94dd-4642-9667-3d33c589a20c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:48:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:15Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce61788b9103 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: e7ea494b-0e1f-4644-9d35-9cfc36d298f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfce619042a558 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 812d1bf9-349a-4d2c-835e-47b70ed634f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:29Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfcee28c15bfa8 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 12eaf79f-d2ff-46e9-a321-640c15beba78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:57:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:29Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfcee28c16dd75 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: d0121546-7d09-41c9-9488-4419eac4e550 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 18578650-14bd-4276-989b-41074feddf54 kind: Event lastTimestamp: "2026-07-06T20:57:31Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:31Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq.18bfcee304d2a88a namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: ee37ac22-d8ab-4dcc-9174-3d60fbc4b650 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-142.ec2.internal source: component: kubelet host: ip-10-0-142-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dfad7-predictor-dffd7f4cb namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: d1ae5a24-96db-410d-b6b2-dbd086f3652d kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: 'Created pod: success-200-isvc-dfad7-predictor-dffd7f4cb-6ddvq' metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:14Z" name: success-200-isvc-dfad7-predictor-dffd7f4cb.18bfce6154e72f92 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 223b204f-8295-4cab-9000-7eada173e94b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: bbcfc034-8a3e-4000-ad7b-3fb3db1a40a1 kind: Event lastTimestamp: "2026-07-06T20:48:14Z" message: Scaled up replica set success-200-isvc-dfad7-predictor-dffd7f4cb from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:14Z" name: success-200-isvc-dfad7-predictor.18bfce61531ce9f9 namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: 24595755-08fa-4b9c-b2b7-e5617a4f30cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: f856086d-d673-4263-8d45-1e266506aaeb kind: Event lastTimestamp: "2026-07-06T20:48:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:44Z" name: success-200-isvc-dfad7-predictor.18bfce64d1afebe2 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 263c75a1-69ef-42b1-a133-8150c1ae58bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: f856086d-d673-4263-8d45-1e266506aaeb kind: Event lastTimestamp: "2026-07-06T20:48:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:44Z" name: success-200-isvc-dfad7-predictor.18bfce64d2140bfe namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: f8580a8e-d44c-4ccd-a9a4-b0b41145703e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:48:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: f856086d-d673-4263-8d45-1e266506aaeb kind: Event lastTimestamp: "2026-07-06T20:49:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:29Z" name: success-200-isvc-dfad7-predictor.18bfce6bd00b5ef4 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: fd58389b-5bd4-42b3-b4dd-da80a975f7e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:48:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dfad7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: f856086d-d673-4263-8d45-1e266506aaeb kind: Event lastTimestamp: "2026-07-06T20:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:29Z" name: success-200-isvc-dfad7-predictor.18bfce6bd080a704 namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: 1b20bd70-a866-48b7-8aa3-944a0f2ae909 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:48:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: e59d59fd-d87f-44ab-8a0a-060572317bc6 kind: Event lastTimestamp: "2026-07-06T20:49:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:02Z" name: success-200-isvc-dfad7.18bfce61531bff53 namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 795af0d0-f070-48f2-8529-dc956184785e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dfad7 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: e59d59fd-d87f-44ab-8a0a-060572317bc6 kind: Event lastTimestamp: "2026-07-06T20:49:02Z" message: InferenceService [success-200-isvc-dfad7] is Ready metadata: creationTimestamp: "2026-07-06T20:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:02Z" name: success-200-isvc-dfad7.18bfce6c4daa81bc namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: 3e3af535-86bc-49c0-b3c3-f898deb271ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T21:08:54.985511Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-32e32-56c9887cf-bgkh9 to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T21:08:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T21:08:54Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfcf8210245017 namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: d130daa1-7581-47e6-aae0-9427ecd854df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:08:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-32e32-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T21:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:55Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfcf821f38d7ed namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 20650e9e-b603-41c8-869f-45f2f7b2a270 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:56Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:08:56Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T21:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T21:08:56Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfcf824d466e1e namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 76428a84-59c4-435c-94cc-64a9aa8701a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32e32} kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:08:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T21:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:56Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfcf824ecbf75f namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: f207da62-ad57-4cb3-9cb1-4f16c7f2875f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32e32} kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:08:56Z" message: 'Created container: switch-graph-32e32' metadata: creationTimestamp: "2026-07-06T21:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:56Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfcf8251e1b991 namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: 731dd440-27d4-4bff-8314-20d963af3531 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32e32} kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:08:56Z" message: Started container switch-graph-32e32 metadata: creationTimestamp: "2026-07-06T21:08:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:08:56Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfcf82524cac10 namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 37a65052-6c9c-433b-a5a0-d977a2b11c01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32e32} kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:25:14Z" message: Stopping container switch-graph-32e32 metadata: creationTimestamp: "2026-07-06T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:25:14Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfd066071b75d6 namespace: kserve-ci-e2e-test resourceVersion: "33767" uid: 94d543c5-9cc6-4161-a1b5-b44a55e29c2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T21:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32e32} kind: Pod name: switch-graph-32e32-56c9887cf-bgkh9 namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 0fa8cd06-104b-485e-897d-850fc8300810 kind: Event lastTimestamp: "2026-07-06T21:25:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T21:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:25:42Z" name: switch-graph-32e32-56c9887cf-bgkh9.18bfd066e8479df5 namespace: kserve-ci-e2e-test resourceVersion: "34044" uid: 24e0b358-621f-46ba-9112-a18551d3ebe6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-32e32-56c9887cf namespace: kserve-ci-e2e-test resourceVersion: "26828" uid: 9bf1df39-c7ed-43e3-b143-8d0fb0e2019a kind: Event lastTimestamp: "2026-07-06T21:08:54Z" message: 'Created pod: switch-graph-32e32-56c9887cf-bgkh9' metadata: creationTimestamp: "2026-07-06T21:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:54Z" name: switch-graph-32e32-56c9887cf.18bfcf820f8703e7 namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: f63bf5ba-d657-47ed-8679-927fbe9e5650 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26826" uid: 915fe640-920e-4e97-ae64-31a447f79c47 kind: Event lastTimestamp: "2026-07-06T21:08:54Z" message: Scaled up replica set switch-graph-32e32-56c9887cf from 0 to 1 metadata: creationTimestamp: "2026-07-06T21:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:08:54Z" name: switch-graph-32e32.18bfcf820e36a7c1 namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 89225ac5-cf5e-44fb-85bd-3926d40d0581 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 1cf28b7f-49c7-406a-bab0-40a85c64daee kind: Event lastTimestamp: "2026-07-06T21:08:55Z" message: InferenceGraph [switch-graph-32e32] is Ready metadata: creationTimestamp: "2026-07-06T21:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:55Z" name: switch-graph-32e32.18bfcf82132c87e0 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: f47d40fb-69bd-46fe-8244-bb3167421c41 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26821" uid: 1cf28b7f-49c7-406a-bab0-40a85c64daee kind: Event lastTimestamp: "2026-07-06T21:08:55Z" message: 'Failed to update status for InferenceGraph "switch-graph-32e32": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-32e32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:55Z" name: switch-graph-32e32.18bfcf8216b95497 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 899f895b-abde-42e1-bce6-88767f077171 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:08:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26821" uid: 1cf28b7f-49c7-406a-bab0-40a85c64daee kind: Event lastTimestamp: "2026-07-06T21:08:55Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-32e32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T21:08:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T21:08:55Z" name: switch-graph-32e32.18bfcf8216b9eaa4 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: c8371df4-0c4f-4519-a318-22892fa30284 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: 43b17033-d444-417b-aa29-7da5509b5dba kind: Event lastTimestamp: "2026-07-06T21:09:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:24Z" name: switch-graph-32e32.18bfcf858cd08e0e namespace: kserve-ci-e2e-test resourceVersion: "27110" uid: d1264ba0-2bdf-4932-ac51-2be0438195c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T21:09:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-32e32 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: 43b17033-d444-417b-aa29-7da5509b5dba kind: Event lastTimestamp: "2026-07-06T21:09:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T21:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T21:09:25Z" name: switch-graph-32e32.18bfcf858d430732 namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: 93b910d6-252a-488d-8499-8165216c3e7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:47:24.620286Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-85420-5d67fb78c6-cxj5w to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce55a06188f7 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: e82d8022-fd66-4c9f-ac53-ad3362d07f33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:24Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-85420-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce55acdaa4f1 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: a90f80e6-71dc-40fb-b9fe-2cdadc53ae63 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:25Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:47:25Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce55dd6e07cf namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 33614ac9-35df-4463-96ae-3948ac6a6783 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85420} kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:25Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" metadata: creationTimestamp: "2026-07-06T20:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:25Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce55dee31631 namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: adffd194-6015-4d89-9258-db07233d71ad reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85420} kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" in 2.119s (2.119s including waiting). Image size: 218071146 bytes.' metadata: creationTimestamp: "2026-07-06T20:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:27Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce565d382a1c namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: d11af1c0-0fde-463c-a039-0bf94f98bbfc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85420} kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:27Z" message: 'Created container: switch-graph-85420' metadata: creationTimestamp: "2026-07-06T20:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:27Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce565ffdee8e namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: fede8074-6d01-444d-8656-9fe2b4933240 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85420} kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:27Z" message: Started container switch-graph-85420 metadata: creationTimestamp: "2026-07-06T20:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:27Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce566069c497 namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 7b31762d-0bd8-457d-8980-f48a58227bb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85420} kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:47:34Z" message: Stopping container switch-graph-85420 metadata: creationTimestamp: "2026-07-06T20:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:34Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce5801e9d9ac namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: c2cb1d34-90f5-40f8-b1de-f0dd8d1a0b98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:47:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-85420} kind: Pod name: switch-graph-85420-5d67fb78c6-cxj5w namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 0602ee6b-3f5e-4601-b2c0-e93758035dfa kind: Event lastTimestamp: "2026-07-06T20:48:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:04Z" name: switch-graph-85420-5d67fb78c6-cxj5w.18bfce590680cf98 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 01b05393-4101-4031-a2e1-ac1823979a4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-85420-5d67fb78c6 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: f088d733-497e-4ce8-87ab-7ee769eb9be2 kind: Event lastTimestamp: "2026-07-06T20:47:24Z" message: 'Created pod: switch-graph-85420-5d67fb78c6-cxj5w' metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420-5d67fb78c6.18bfce559fc76f1f namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: dc5baf5e-cf9b-44d6-90e0-70ea814ec6a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-85420 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 67621efd-2e11-43a7-8810-785d736f0ee1 kind: Event lastTimestamp: "2026-07-06T20:47:24Z" message: Scaled up replica set switch-graph-85420-5d67fb78c6 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420.18bfce559ea0f3cc namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 3f96a0d0-c66c-4038-88de-5ede6a10c192 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-85420 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: e66d3d4a-c4a1-4602-8267-a0d87d7c6b9e kind: Event lastTimestamp: "2026-07-06T20:47:24Z" message: InferenceGraph [switch-graph-85420] is Ready metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420.18bfce55a302efb0 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: a845f75a-d0e9-40d2-b8e2-b3973ea9c2fe reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-85420 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: e66d3d4a-c4a1-4602-8267-a0d87d7c6b9e kind: Event lastTimestamp: "2026-07-06T20:47:24Z" message: 'Failed to update status for InferenceGraph "switch-graph-85420": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-85420": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420.18bfce55a64259dd namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 91dacb2b-31dd-4ead-ae29-1c3492989bf1 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-85420 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: e66d3d4a-c4a1-4602-8267-a0d87d7c6b9e kind: Event lastTimestamp: "2026-07-06T20:47:24Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-85420": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:24Z" name: switch-graph-85420.18bfce55a642a905 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 77a7ff1a-0ef4-46a8-a463-085932a59d56 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:35.102614Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-87d4d-6dfd87976f-kww7r to ip-10-0-132-244.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfce6609748723 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: a0fee035-79f4-4fa3-8b41-66296ca19166 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7649f967f8-tr2kd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfce662277adb0 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 57d42e94-1fe8-44c1-a52d-87464335f500 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87d4d} kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfce6623dfef5a namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 30089c07-bbba-4d6b-b10f-ddfde3867dd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87d4d} kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: 'Created container: switch-graph-87d4d' metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfce6626da44d9 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: b8f08c33-3d0f-40ad-98ae-1e88f4c2bf22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87d4d} kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: Started container switch-graph-87d4d metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfce6627428888 namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: 94bfbd25-b4d8-4852-a157-6f4b80686c9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87d4d} kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: "2026-07-06T20:56:49Z" message: Stopping container switch-graph-87d4d metadata: creationTimestamp: "2026-07-06T20:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:49Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfced934deefd9 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 945ffd02-621b-44d5-8575-c76a72d90cec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87d4d} kind: Pod name: switch-graph-87d4d-6dfd87976f-kww7r namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 498c6f24-ef95-4fe0-9ae9-b8d93076b086 kind: Event lastTimestamp: "2026-07-06T20:57:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:57:17Z" name: switch-graph-87d4d-6dfd87976f-kww7r.18bfced9d809a8a7 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: cc9f0eca-4eef-47ed-b4b5-334bba670c22 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-244.ec2.internal source: component: kubelet host: ip-10-0-132-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-87d4d-6dfd87976f namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: d3c1a7ca-bc19-4278-9560-190c63340d8c kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: 'Created pod: switch-graph-87d4d-6dfd87976f-kww7r' metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d-6dfd87976f.18bfce66089b8489 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: f8542cf9-7402-419d-b0ae-86e6967fbcd4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 18cb0a43-ad2b-433c-a036-dc17286ff388 kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: Scaled up replica set switch-graph-87d4d-6dfd87976f from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d.18bfce660740e634 namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: 7c5f6198-3aa2-402a-ada5-d2d9971019b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: a7b042db-bb1d-463f-a29e-6e91ebe2be3a kind: Event lastTimestamp: "2026-07-06T20:48:35Z" message: InferenceGraph [switch-graph-87d4d] is Ready metadata: creationTimestamp: "2026-07-06T20:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:35Z" name: switch-graph-87d4d.18bfce660c8bcba8 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 4ece3460-0015-4426-929f-ee7bbd79fc3d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: f3f493c7-a851-495c-b077-2089bac8b3cd kind: Event lastTimestamp: "2026-07-06T20:48:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:50Z" name: switch-graph-87d4d.18bfce6985d24d72 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: ab18bc06-357a-4f27-9f3b-1f4eece1a0b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: f3f493c7-a851-495c-b077-2089bac8b3cd kind: Event lastTimestamp: "2026-07-06T20:48:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:50Z" name: switch-graph-87d4d.18bfce69864fd78c namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: a8ce727a-1048-4496-85a5-abbd553771b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: f3f493c7-a851-495c-b077-2089bac8b3cd kind: Event lastTimestamp: "2026-07-06T20:49:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:05Z" name: switch-graph-87d4d.18bfce6d05113849 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 130a8a88-993a-478f-bd8b-a6fe32c81823 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87d4d namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: f3f493c7-a851-495c-b077-2089bac8b3cd kind: Event lastTimestamp: "2026-07-06T20:49:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:05Z" name: switch-graph-87d4d.18bfce6d058f65cb namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 409dd9bb-c2c9-48b8-a996-c505994a5fbf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34045"