--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:33:18.255616Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-3e634-5f7d5cf85c-d6nd4 to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc092ce76dc namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 90c15af8-ff0e-4048-bf3a-1f29923c308a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc0b4c10afb namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 25f927bf-4fbb-4080-ae12-732bc835d6e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e634} kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc0b632bbed namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: faf1caf9-5c6e-4d69-ab3d-d073f49f57d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e634} kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: 'Created container: ensemble-graph-3e634' metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc0b9257726 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: baf04d56-900d-4232-a5b1-4414690bb131 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e634} kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: Started container ensemble-graph-3e634 metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc0b99585e8 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 57d14573-ddc7-46d3-9378-e44e05fc7a58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e634} kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Stopping container ensemble-graph-3e634 metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:28Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc2eb1da56a namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: 52643dd7-7141-484c-a526-214b8dae33b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e634} kind: Pod name: ensemble-graph-3e634-5f7d5cf85c-d6nd4 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 8cffc0f5-ef77-4e6d-a972-dd18eb131c6d kind: Event lastTimestamp: "2026-09-17T23:33:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:55Z" name: ensemble-graph-3e634-5f7d5cf85c-d6nd4.18d63fc35802fce2 namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: 2f7f1c2a-ed5c-48ce-b0de-7b4690dd6290 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-3e634-5f7d5cf85c namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 3dd183b5-e0ec-46f2-accd-b03e35e60e8b kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: 'Created pod: ensemble-graph-3e634-5f7d5cf85c-d6nd4' metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634-5f7d5cf85c.18d63fc09266f153 namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 0f74d096-2698-490c-a16d-a14df8e5565d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-3e634 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 4f87616a-2c0c-4768-a440-f4dd0801293d kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: Scaled up replica set ensemble-graph-3e634-5f7d5cf85c from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634.18d63fc091a51b61 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 2f1b2638-2aa2-4be4-9b89-70509b6c20d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3e634 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: 4c19b3be-e69a-4723-a19a-cae69005c53c kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: InferenceGraph [ensemble-graph-3e634] is Ready metadata: creationTimestamp: "2026-09-17T23:33:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:18Z" name: ensemble-graph-3e634.18d63fc09705b262 namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 2852ee61-98cf-4057-ba01-16a5ec3fd28f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:34:28.594738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-d6263-569646bdf9-hgph9 to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:34:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:34:28Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d63fd0f3585eea namespace: kserve-ci-e2e-test resourceVersion: "17691" uid: dd8fe9ee-a499-4af5-a7f2-c44612d20c1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:29Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17695" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: "2026-09-17T23:34:29Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:34:29Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d63fd11569dccc namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: db49c41b-204c-401b-a962-48587f312a85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6263} kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: "2026-09-17T23:34:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:29Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d63fd116c9c7af namespace: kserve-ci-e2e-test resourceVersion: "17708" uid: 15a811d4-2153-49b1-a3c3-06490f3895dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6263} kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: "2026-09-17T23:34:29Z" message: 'Created container: ensemble-graph-d6263' metadata: creationTimestamp: "2026-09-17T23:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:29Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d63fd11a159a20 namespace: kserve-ci-e2e-test resourceVersion: "17709" uid: 65ad86ec-9c91-4cbb-b3d7-338175272238 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6263} kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: "2026-09-17T23:34:29Z" message: Started container ensemble-graph-d6263 metadata: creationTimestamp: "2026-09-17T23:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:29Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d63fd11a88256b namespace: kserve-ci-e2e-test resourceVersion: "17710" uid: f8548439-ded8-4add-ad96-e854f9c8ea49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6263} kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Stopping container ensemble-graph-d6263 metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:43Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d64044236e0d1c namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: 3e58ec43-9f26-4a73-a97d-bdc0b1321f21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6263} kind: Pod name: ensemble-graph-d6263-569646bdf9-hgph9 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: a0752949-cb2d-4408-ab04-40695b6fc449 kind: Event lastTimestamp: "2026-09-17T23:43:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:10Z" name: ensemble-graph-d6263-569646bdf9-hgph9.18d64044a5e3fe28 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: e89a7d84-debd-49d5-b4f4-78c4496397ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-d6263-569646bdf9 namespace: kserve-ci-e2e-test resourceVersion: "17681" uid: 541bb42a-a8ea-480d-a843-88c8f2d47ee3 kind: Event lastTimestamp: "2026-09-17T23:34:28Z" message: 'Created pod: ensemble-graph-d6263-569646bdf9-hgph9' metadata: creationTimestamp: "2026-09-17T23:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:28Z" name: ensemble-graph-d6263-569646bdf9.18d63fd0f2de64ea namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: 063fabd1-46de-40cb-a838-bc7f6ab798c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17679" uid: b35388b4-5dba-49c6-8719-72680fa7969d kind: Event lastTimestamp: "2026-09-17T23:34:28Z" message: Scaled up replica set ensemble-graph-d6263-569646bdf9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:28Z" name: ensemble-graph-d6263.18d63fd0f202d888 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: 3db9b293-14e3-4b8f-9ea7-1e9c0d86fed3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17697" uid: bda4fa6b-01f2-4f37-9b8f-69e8e1e0e938 kind: Event lastTimestamp: "2026-09-17T23:34:28Z" message: InferenceGraph [ensemble-graph-d6263] is Ready metadata: creationTimestamp: "2026-09-17T23:34:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:34:28Z" name: ensemble-graph-d6263.18d63fd0f7fe266e namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: 4cc4025c-4d19-4cc4-ad75-b520c045178f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 924b120e-4eca-4bac-bc0e-62516b39b243 kind: Event lastTimestamp: "2026-09-17T23:34:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:43Z" name: ensemble-graph-d6263.18d63fd47116fd8c namespace: kserve-ci-e2e-test resourceVersion: "17799" uid: 855437aa-9a35-420f-a321-c6eb0c95597e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17680" uid: 924b120e-4eca-4bac-bc0e-62516b39b243 kind: Event lastTimestamp: "2026-09-17T23:34:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:43Z" name: ensemble-graph-d6263.18d63fd4717e9369 namespace: kserve-ci-e2e-test resourceVersion: "17800" uid: 3893fcc8-21b5-4e20-bf1a-4f6eada8a6cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17796" uid: 924b120e-4eca-4bac-bc0e-62516b39b243 kind: Event lastTimestamp: "2026-09-17T23:34:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:58Z" name: ensemble-graph-d6263.18d63fd7f052dae3 namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: 911ae087-af77-466d-824d-531981780d76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17796" uid: 924b120e-4eca-4bac-bc0e-62516b39b243 kind: Event lastTimestamp: "2026-09-17T23:34:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:58Z" name: ensemble-graph-d6263.18d63fd7f0d7ec55 namespace: kserve-ci-e2e-test resourceVersion: "17879" uid: 9e99cfc2-cf78-4f62-98cf-ff0f24442ad0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:23:03.541218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13318" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:23:03Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f3172f91a12 namespace: kserve-ci-e2e-test resourceVersion: "13322" uid: 38b9f491-891b-43f4-b059-d7bf7ca63e66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13328" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f319533c28e namespace: kserve-ci-e2e-test resourceVersion: "13348" uid: d3e4ede9-9c6f-4b84-a87f-1adfacc738f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f3196cdc1fa namespace: kserve-ci-e2e-test resourceVersion: "13350" uid: d05ec5b6-e4d7-4523-b127-4e13530f2e74 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f31998e8ee5 namespace: kserve-ci-e2e-test resourceVersion: "13353" uid: c58b8e55-995c-45cc-ab6c-0c2450dd0c77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f3199fe3b6f namespace: kserve-ci-e2e-test resourceVersion: "13354" uid: 883b9ea2-b52f-4790-97a1-0afbcb6d9240 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f319a09583a namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: a4bec310-2305-438a-88dd-2c9fa3d10ce7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f319d05c17e namespace: kserve-ci-e2e-test resourceVersion: "13356" uid: bd95c7fc-5b47-4889-ba4d-414d2b167621 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f319dbd3c2d namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: eeb6cffc-6909-4e57-889c-db8d3f903716 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:23:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:41Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63f31f83199c0 namespace: kserve-ci-e2e-test resourceVersion: "13639" uid: fb2d9c87-2ac5-4706-ae18-6fcd70b513b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63fb295c6dd04 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 698e5383-03b7-4346-bde1-858a27b9a197 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4 namespace: kserve-ci-e2e-test resourceVersion: "13320" uid: fb23a376-b313-42e2-82cb-2ae293396b0f kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4.18d63fb295c7e249 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 5938c36f-4390-4746-ab8d-4b6d3679d59f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-11096-predictor-5d7bddccd5 namespace: kserve-ci-e2e-test resourceVersion: "13311" uid: 4f3bb182-aca7-4e29-92fe-9f7ebd9cf7f0 kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: 'Created pod: error-404-isvc-11096-predictor-5d7bddccd5-6nvn4' metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:03Z" name: error-404-isvc-11096-predictor-5d7bddccd5.18d63f3172231139 namespace: kserve-ci-e2e-test resourceVersion: "13319" uid: ba4611e3-6edf-4a94-85dc-cca285008cc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13307" uid: 0a540df7-d0eb-4b51-9f49-1b7d510ec0bd kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Scaled up replica set error-404-isvc-11096-predictor-5d7bddccd5 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:03Z" name: error-404-isvc-11096-predictor.18d63f3170288cb5 namespace: kserve-ci-e2e-test resourceVersion: "13312" uid: 0abfa0f5-9672-476f-90cd-e79f6e900cc5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: e352c115-831c-45b0-8acb-bc415b1c16d9 kind: Event lastTimestamp: "2026-09-17T23:23:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:18Z" name: error-404-isvc-11096-predictor.18d63f34ee920ee6 namespace: kserve-ci-e2e-test resourceVersion: "13470" uid: a4eb1b3b-d41a-4b56-8d4d-58afc597608b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13310" uid: e352c115-831c-45b0-8acb-bc415b1c16d9 kind: Event lastTimestamp: "2026-09-17T23:23:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:18Z" name: error-404-isvc-11096-predictor.18d63f34eefa4c53 namespace: kserve-ci-e2e-test resourceVersion: "13472" uid: e10b80c5-6505-46f1-89e6-dc1caba6cc9c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:23:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13471" uid: e352c115-831c-45b0-8acb-bc415b1c16d9 kind: Event lastTimestamp: "2026-09-17T23:24:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:18Z" name: error-404-isvc-11096-predictor.18d63f386da97893 namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: a4ba794c-0f1d-4e45-a3ea-d5705e02ea14 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:23:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13471" uid: e352c115-831c-45b0-8acb-bc415b1c16d9 kind: Event lastTimestamp: "2026-09-17T23:24:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:18Z" name: error-404-isvc-11096-predictor.18d63f386e15d1bf namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: 5c76a0fc-69b8-467a-a8ff-f85b8c418426 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11096 namespace: kserve-ci-e2e-test resourceVersion: "13294" uid: 3290990a-fccc-4632-8e7f-5c5a2009b047 kind: Event lastTimestamp: "2026-09-17T23:23:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:51Z" name: error-404-isvc-11096.18d63f3170288b8a namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 1dde377c-acf6-4c88-891b-b5b7289335d9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11096 namespace: kserve-ci-e2e-test resourceVersion: "13863" uid: 3290990a-fccc-4632-8e7f-5c5a2009b047 kind: Event lastTimestamp: "2026-09-17T23:23:51Z" message: InferenceService [error-404-isvc-11096] is Ready metadata: creationTimestamp: "2026-09-17T23:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:51Z" name: error-404-isvc-11096.18d63f3cb43b4700 namespace: kserve-ci-e2e-test resourceVersion: "13864" uid: d87b0724-548f-49fb-8219-ee1222a1d078 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:23:50.210307Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13806" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c50aaf303 namespace: kserve-ci-e2e-test resourceVersion: "13811" uid: e8e9970c-bd0d-4222-9fd5-3b7968ef59a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13816" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c7284a800 namespace: kserve-ci-e2e-test resourceVersion: "13837" uid: 70b02ddf-a03e-4f7a-9ccf-854983f7330d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c7486636a namespace: kserve-ci-e2e-test resourceVersion: "13839" uid: 90ac921e-5703-424d-b36b-9c6ddbff300a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c777c7086 namespace: kserve-ci-e2e-test resourceVersion: "13841" uid: faec504f-7978-4004-a6ff-ae0083f3a7f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c77e6536f namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: 5197c263-5637-4d51-ae82-67ae7fee7506 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c77ef1c77 namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: ee29a6f2-5558-4376-afab-f5c1de76e622 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c7a8c9bbf namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: 7c1c9181-8e85-4c25-baf6-6cdc51cf2d03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3c7af71114 namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: b9fbd9d9-648a-426f-bfb4-7f20a81e931c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:24:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:28Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63f3cbc2f0289 namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 6768a90b-e4a5-42e1-9e2c-feba1625a2fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:04Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63fbd65571d16 namespace: kserve-ci-e2e-test resourceVersion: "16699" uid: 5bbba0c9-bcdc-4b5f-91e9-7d5c65ae7f61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: ec6ad9c8-ca76-4742-b02c-44393a72e6b6 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:04Z" name: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq.18d63fbd65578aae namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: b6861048-3bf9-4846-ae16-4693cad86212 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2026c-predictor-5f7545dbd9 namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: bc2dcbed-4b97-4738-95f7-d98f20e9b05d kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: 'Created pod: error-404-isvc-2026c-predictor-5f7545dbd9-9k6vq' metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor-5f7545dbd9.18d63f3c4fe99cd7 namespace: kserve-ci-e2e-test resourceVersion: "13808" uid: bed64fa7-d710-4793-ad12-f9b338343aec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13798" uid: 4793a951-a048-4e24-99ae-c7b871e1ccb7 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Scaled up replica set error-404-isvc-2026c-predictor-5f7545dbd9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:50Z" name: error-404-isvc-2026c-predictor.18d63f3c4d62291d namespace: kserve-ci-e2e-test resourceVersion: "13803" uid: be1ca37e-372c-4e4c-8873-829055b9e58b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:24:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: 3f5dd9ce-7b15-4bd9-9390-1dd345142b62 kind: Event lastTimestamp: "2026-09-17T23:24:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:35Z" name: error-404-isvc-2026c-predictor.18d63f3fcbe1f9fe namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 1eaeada4-fcbd-4107-8e99-030da6d4ad95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:24:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: 3f5dd9ce-7b15-4bd9-9390-1dd345142b62 kind: Event lastTimestamp: "2026-09-17T23:24:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:35Z" name: error-404-isvc-2026c-predictor.18d63f3fccb3ef38 namespace: kserve-ci-e2e-test resourceVersion: "14167" uid: b461800e-c6d0-40ce-82e6-f08420501a50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: 3f5dd9ce-7b15-4bd9-9390-1dd345142b62 kind: Event lastTimestamp: "2026-09-17T23:25:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:25:05Z" name: error-404-isvc-2026c-predictor.18d63f434b534471 namespace: kserve-ci-e2e-test resourceVersion: "14373" uid: a2c78537-f057-4d47-83f5-1d973dca49c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: 3f5dd9ce-7b15-4bd9-9390-1dd345142b62 kind: Event lastTimestamp: "2026-09-17T23:25:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:24:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:25:05Z" name: error-404-isvc-2026c-predictor.18d63f434bb81d29 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: 7978f4b4-f973-45a1-b8cd-6ecf61c47a9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2026c namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 03055cd6-44ac-4c21-8915-aea3d34686c1 kind: Event lastTimestamp: "2026-09-17T23:24:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:38Z" name: error-404-isvc-2026c.18d63f3c4d31cda0 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 1453a82d-7d60-47c2-80a2-6f6a0306d41b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2026c namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: 03055cd6-44ac-4c21-8915-aea3d34686c1 kind: Event lastTimestamp: "2026-09-17T23:24:38Z" message: InferenceService [error-404-isvc-2026c] is Ready metadata: creationTimestamp: "2026-09-17T23:24:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:38Z" name: error-404-isvc-2026c.18d63f477aab37af namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 37b1e7ca-8f40-493c-a0e3-074ff939fae6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:32:18.358436Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3e634-predictor-78db59dc7-blq7n to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:32:18Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2a0a8065f namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 1725cbe8-822a-4ed3-afd9-d82d5826c5e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3e634-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2a76c74cd namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 11e1eabd-024e-4d87-8f7b-8dec6431d459 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2e6948d4c namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 6df57203-b096-469a-b8f8-068f20c5bd54 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2e86c8611 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 8e55776e-3b18-4e10-9d19-9d76d59fd52d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2ebb48cc5 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 250eab98-787c-463d-912c-53ce00cbcbfc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2ec225f85 namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 0db44c8e-aa1a-48d6-bfd9-f261d56933d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2ec2ba95b namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: d05b46fc-e215-4501-a37a-be261c94b231 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2eef39b4a namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 8eade317-a6ad-4df5-91a4-77eecff1c191 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb2ef5d9f96 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 1d6276ce-3c80-443c-87f3-faef17f26bae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:32:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:57Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fb3545345e4 namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 793ad365-9ee7-4c7f-b97b-8724780fbb1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:28Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fc2f422a986 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: d0e11633-aaae-4097-ab35-3e0924cac5b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e634-predictor-78db59dc7-blq7n namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 4a19abef-acb7-4683-9c35-7563bb6cb8f4 kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:28Z" name: error-404-isvc-3e634-predictor-78db59dc7-blq7n.18d63fc2f4231c7e namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: aa882398-4e49-44da-b4b8-8a2a9f1378a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3e634-predictor-78db59dc7 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 3ee75314-c55e-4c88-b726-a8d187ca1ad7 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: 'Created pod: error-404-isvc-3e634-predictor-78db59dc7-blq7n' metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:32:18Z" name: error-404-isvc-3e634-predictor-78db59dc7.18d63fb2a00596b6 namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 84e95373-8b59-4056-b5bd-3362d87fde85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: bc45bce0-368b-45f1-b3c6-5be3b959f150 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Scaled up replica set error-404-isvc-3e634-predictor-78db59dc7 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:32:18Z" name: error-404-isvc-3e634-predictor.18d63fb29e5f3523 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 61fb33e8-d7d5-4f03-a51f-9998bf0b2952 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 01df3046-686e-44e6-aa20-724b0c705dbd kind: Event lastTimestamp: "2026-09-17T23:33:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:03Z" name: error-404-isvc-3e634-predictor.18d63fb61cc30fc7 namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 15c97867-f162-47bc-ae98-1831cb594d4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 01df3046-686e-44e6-aa20-724b0c705dbd kind: Event lastTimestamp: "2026-09-17T23:33:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:03Z" name: error-404-isvc-3e634-predictor.18d63fb61d06e4cb namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: fb2e7c7d-6d21-4e6d-a544-5f610786f42e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 01df3046-686e-44e6-aa20-724b0c705dbd kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:18Z" name: error-404-isvc-3e634-predictor.18d63fb99ba3ec3a namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 7f10a0c8-05d4-4acc-b8d9-c70d65df5b11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 01df3046-686e-44e6-aa20-724b0c705dbd kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:18Z" name: error-404-isvc-3e634-predictor.18d63fb99c0eeceb namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 2836a9bc-5a51-4821-9e5c-8b3421276bee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e634 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 0bfa84cf-5df6-4912-bd1d-d8c2a5e92fdf kind: Event lastTimestamp: "2026-09-17T23:33:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:07Z" name: error-404-isvc-3e634.18d63fb29e60b670 namespace: kserve-ci-e2e-test resourceVersion: "16814" uid: d3849bc0-383a-43b2-af43-93cf8e1c4e38 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e634 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 0bfa84cf-5df6-4912-bd1d-d8c2a5e92fdf kind: Event lastTimestamp: "2026-09-17T23:33:07Z" message: InferenceService [error-404-isvc-3e634] is Ready metadata: creationTimestamp: "2026-09-17T23:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:07Z" name: error-404-isvc-3e634.18d63fbe114264d4 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: 0c9a3cb8-2642-49e6-9cc3-edb3209f3fda reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:33:04.815925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:33:04Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd71bd0374 namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: c8e7f1a0-5579-4d20-9e4c-6cef440be00b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd938ba406 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 0c863386-09b7-4a34-999c-3279f204a387 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd95ef9408 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: d07c104b-b662-4b67-9f00-a2fb903acbdd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd98fba532 namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: 28992eb8-bcda-479d-83a0-facfb95d65d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd99cef41c namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: c564c204-e2f8-4ca6-93fa-278380fb73ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd99d93b79 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: 648645db-7756-42a6-adf1-e315a240676f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd9d400e10 namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 8ea8f931-42e3-4d5e-9471-6ad3744af9b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbd9daa91e8 namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: 398015ba-96a4-498f-8d6e-be6dbeccfdcd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:33:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:42Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fbddb27eb22 namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: 23e70f25-6820-4c90-b6d2-26ff82d03047 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:14Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fcdc39ab9b3 namespace: kserve-ci-e2e-test resourceVersion: "17498" uid: acc62f7f-aadc-4250-83cd-d85db66f453b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:14Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fcdc39b5a05 namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: 230a76d1-f5e8-4363-8fcc-a1fabdd55a64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: e78fdfc0-af40-40ba-8682-fb61e1ef4866 kind: Event lastTimestamp: "2026-09-17T23:34:17Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:17Z" name: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5.18d63fce637e89fd namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: 5f8bf218-5064-43e3-b499-7ee1dc9d13cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4a1f9-predictor-555b74f697 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 9bafab1e-240e-4b2b-aea6-4f384a082177 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: 'Created pod: error-404-isvc-4a1f9-predictor-555b74f697-hp8h5' metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:04Z" name: error-404-isvc-4a1f9-predictor-555b74f697.18d63fbd7146709b namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 8f87b692-005e-496c-ab35-e5afd0ac9a82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 0ba2d25a-f6fc-4402-87d9-16df9291de87 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Scaled up replica set error-404-isvc-4a1f9-predictor-555b74f697 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:04Z" name: error-404-isvc-4a1f9-predictor.18d63fbd6f285e95 namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: 227c839b-08f1-4e5e-ae5e-23da83be5e18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 13e41c39-e72d-485c-adf9-d5d6038df00f kind: Event lastTimestamp: "2026-09-17T23:33:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:49Z" name: error-404-isvc-4a1f9-predictor.18d63fc0ed97fd29 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 2a5ed561-a55f-43fd-b805-d69fffb25eef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 13e41c39-e72d-485c-adf9-d5d6038df00f kind: Event lastTimestamp: "2026-09-17T23:33:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:49Z" name: error-404-isvc-4a1f9-predictor.18d63fc0edfdbc68 namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 7cd4fa2c-fa9e-486a-97ac-a86d72b347b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: 13e41c39-e72d-485c-adf9-d5d6038df00f kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:04Z" name: error-404-isvc-4a1f9-predictor.18d63fc46caaaee2 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 1d828270-eb1d-4643-94aa-68d0382e8503 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: 13e41c39-e72d-485c-adf9-d5d6038df00f kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:04Z" name: error-404-isvc-4a1f9-predictor.18d63fc46d139cf3 namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: cd192ed7-953d-4794-812a-fbecabc13cb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a1f9 namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: afa0fb1a-3764-4cf0-a348-c59c17d4c640 kind: Event lastTimestamp: "2026-09-17T23:33:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:52Z" name: error-404-isvc-4a1f9.18d63fbd6f26c984 namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 5065702e-6095-46cc-9318-10ba215a1bc9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a1f9 namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: afa0fb1a-3764-4cf0-a348-c59c17d4c640 kind: Event lastTimestamp: "2026-09-17T23:33:52Z" message: InferenceService [error-404-isvc-4a1f9] is Ready metadata: creationTimestamp: "2026-09-17T23:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:52Z" name: error-404-isvc-4a1f9.18d63fc8967e4206 namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: 3f663750-d770-4d7c-be5c-38837608bea6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:34:15.096375Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17537" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-51963-predictor-5b7dc99599-gr8vf to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:34:15Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fcdcec79890 namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: cda64b7a-50ac-45cc-b159-5cda3b92575e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-51963-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:15Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fcddecb2c67 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 5b7c2fcd-9857-4cec-be01-6c262b82933b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce14b42906 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: 06ad5893-77a8-40e2-886d-102089d0bcb3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce1613b346 namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 49a5290a-5c58-4436-a388-f87dfdb86d1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce18ee843d namespace: kserve-ci-e2e-test resourceVersion: "17586" uid: 1bcd8128-a5f4-4dfe-81b8-5cbc408257e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce19620b9c namespace: kserve-ci-e2e-test resourceVersion: "17587" uid: 85f28f61-39f0-497d-a2f4-6ee1b9c677a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce196d0aa8 namespace: kserve-ci-e2e-test resourceVersion: "17588" uid: 848d649e-8aa1-44c6-90b1-9302a94bdf39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce1c217cc0 namespace: kserve-ci-e2e-test resourceVersion: "17589" uid: e72634f6-2a23-4796-a21f-9455f713824c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce1c92e47e namespace: kserve-ci-e2e-test resourceVersion: "17590" uid: 405314ab-a7cc-4465-b123-1d83add77df6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:34:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:34:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:53Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d63fce76d968e4 namespace: kserve-ci-e2e-test resourceVersion: "17854" uid: bcc66093-94d0-405d-a0a8-6abb8d8c0068 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:29Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d6404efc52fbd0 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: fa2b73a9-dcfb-48d5-a760-1ceddab2e93d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: f217de27-ace7-42db-95e9-c8889c017890 kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:29Z" name: error-404-isvc-51963-predictor-5b7dc99599-gr8vf.18d6404efc537502 namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 9f56abd3-5700-4e93-b8d7-70f8b73aa778 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-51963-predictor-5b7dc99599 namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: 4a5affd2-e3b0-4383-9308-fa525526b9a3 kind: Event lastTimestamp: "2026-09-17T23:34:15Z" message: 'Created pod: error-404-isvc-51963-predictor-5b7dc99599-gr8vf' metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:15Z" name: error-404-isvc-51963-predictor-5b7dc99599.18d63fcdce531bc7 namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 8ac863b6-6ac5-46c8-82e2-d5e4382fac14 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: 0955326d-6249-4e1f-882a-86da144759d2 kind: Event lastTimestamp: "2026-09-17T23:34:15Z" message: Scaled up replica set error-404-isvc-51963-predictor-5b7dc99599 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:15Z" name: error-404-isvc-51963-predictor.18d63fcdcc8bc0c5 namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 014a7e5e-4dae-41b8-91ad-c325335fde3c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: a4646d12-1899-4498-a96a-d98034fa4e58 kind: Event lastTimestamp: "2026-09-17T23:34:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:30Z" name: error-404-isvc-51963-predictor.18d63fd14b041063 namespace: kserve-ci-e2e-test resourceVersion: "17724" uid: 29580bd9-3a68-41f7-ae0e-b1f0e27d95f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: a4646d12-1899-4498-a96a-d98034fa4e58 kind: Event lastTimestamp: "2026-09-17T23:34:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:30Z" name: error-404-isvc-51963-predictor.18d63fd14b6daab1 namespace: kserve-ci-e2e-test resourceVersion: "17726" uid: 845ef0c9-8e17-4404-b1ba-33b04cf5f5a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17725" uid: a4646d12-1899-4498-a96a-d98034fa4e58 kind: Event lastTimestamp: "2026-09-17T23:35:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:45Z" name: error-404-isvc-51963-predictor.18d63fd4ca1632ff namespace: kserve-ci-e2e-test resourceVersion: "18163" uid: e2d57d2d-3f92-4941-93a0-3cf87e2f18cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17725" uid: a4646d12-1899-4498-a96a-d98034fa4e58 kind: Event lastTimestamp: "2026-09-17T23:35:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:45Z" name: error-404-isvc-51963-predictor.18d63fd4ca82e988 namespace: kserve-ci-e2e-test resourceVersion: "18164" uid: f8895952-7a7a-449a-a59c-2a60a10e1dec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:34:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-51963 namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: 8a22576b-4d31-40a7-8cec-5a217d6a1b44 kind: Event lastTimestamp: "2026-09-17T23:35:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:35:04Z" name: error-404-isvc-51963.18d63fcdcc8a5df0 namespace: kserve-ci-e2e-test resourceVersion: "17931" uid: dad5879c-9a7a-496d-bb14-90b2e0704aba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-51963 namespace: kserve-ci-e2e-test resourceVersion: "17929" uid: 8a22576b-4d31-40a7-8cec-5a217d6a1b44 kind: Event lastTimestamp: "2026-09-17T23:35:04Z" message: InferenceService [error-404-isvc-51963] is Ready metadata: creationTimestamp: "2026-09-17T23:35:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:35:04Z" name: error-404-isvc-51963.18d63fd93222a4c9 namespace: kserve-ci-e2e-test resourceVersion: "17930" uid: c5ebcef3-4d51-459e-974a-d204e63072cd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:43:30.098248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-57566-predictor-85dbbdfff7-k9s8n to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f07784574 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bd107bbc-517f-4cd9-9559-472fe7072ebb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f29a21b1f namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bb96c013-a946-4cbb-80bd-339a63bb6dc4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f2b194f8c namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 225cd847-21f8-40d1-99f5-fbefbbf43c69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f2e75e307 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 3329ce52-7aa0-4330-b67c-f79ff29224d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f2ee6c480 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 594a2efb-4607-47b7-8a64-d381af95d995 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f2ef0cd95 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 43831527-8126-4cf9-b8d3-05917ad865e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f31f1a75e namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: 28c2254f-fc9f-4c3c-bedb-9348eeee58f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f325f362a namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: ac93c844-7eb4-47e9-bb24-cdb2ca13af84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-17T23:44:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:08Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d6404f93fd7ebe namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: 12bc02a4-5163-4228-8b12-14e983283b60 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T00:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-18T00:00:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T00:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T00:00:59Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d641435d63136d namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: f59ea11f-c2f5-4287-bff6-fff4e390c53d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T00:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: d77b7875-e9e9-4a15-8f78-16f94410a879 kind: Event lastTimestamp: "2026-09-18T00:00:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T00:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T00:00:59Z" name: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n.18d641435d63abe4 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 288b8309-f1ee-4860-b8ec-d291a21c65d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-57566-predictor-85dbbdfff7 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 88606edc-c67c-40ee-a793-32e6524d4304 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: 'Created pod: error-404-isvc-57566-predictor-85dbbdfff7-k9s8n' metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor-85dbbdfff7.18d6404f0710676f namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 29063b98-6f23-4fdc-bea9-43e6d590c247 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: acaede87-6d4c-4fff-bff5-fe320a468047 kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: Scaled up replica set error-404-isvc-57566-predictor-85dbbdfff7 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:30Z" name: error-404-isvc-57566-predictor.18d6404f05582633 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 9c2b359a-aab0-4226-9b1f-1dc2e05a146a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: c8b1285c-975c-4172-b825-dc16263e31fd kind: Event lastTimestamp: "2026-09-17T23:43:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:45Z" name: error-404-isvc-57566-predictor.18d6405283cd9ab7 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 134a11a8-fd3b-4ba1-ae1d-6b93f67e45cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: c8b1285c-975c-4172-b825-dc16263e31fd kind: Event lastTimestamp: "2026-09-17T23:43:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:45Z" name: error-404-isvc-57566-predictor.18d64052843a2df9 namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 36134251-fe27-4e33-9def-5254c01f740d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:44:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: c8b1285c-975c-4172-b825-dc16263e31fd kind: Event lastTimestamp: "2026-09-17T23:44:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:45Z" name: error-404-isvc-57566-predictor.18d64056036b5aab namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 7a59fb73-795f-4a24-aeb2-c8559cd95795 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:44:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: c8b1285c-975c-4172-b825-dc16263e31fd kind: Event lastTimestamp: "2026-09-17T23:44:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:45Z" name: error-404-isvc-57566-predictor.18d640560428c733 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: fe08eb54-785e-4b2f-a8e6-528f418ccf54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57566 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: ec6208b8-ae85-44ec-a233-4ed4c038fcdb kind: Event lastTimestamp: "2026-09-17T23:44:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:18Z" name: error-404-isvc-57566.18d6404f055de821 namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: c8da39a2-43e7-49f0-9a77-55a50894b659 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57566 namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: ec6208b8-ae85-44ec-a233-4ed4c038fcdb kind: Event lastTimestamp: "2026-09-17T23:44:18Z" message: InferenceService [error-404-isvc-57566] is Ready metadata: creationTimestamp: "2026-09-17T23:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:18Z" name: error-404-isvc-57566.18d6405a4fdc5f00 namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: a7423bc9-e68e-4de6-b63e-4c51f21e3e9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:21:29.424532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12481" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b6ba0-predictor-6665558fb9-99hnd to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:21:29Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1b892e6434 namespace: kserve-ci-e2e-test resourceVersion: "12485" uid: dba97272-99ce-4e46-8a16-509796d38df1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12489" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:21:30Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1bac91e640 namespace: kserve-ci-e2e-test resourceVersion: "12542" uid: 67f25dba-7595-4acd-906b-98f4936057db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1bae07192d namespace: kserve-ci-e2e-test resourceVersion: "12544" uid: 40808cae-32c1-45b9-b4e0-6c05cc3e8877 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" in 16.327s (16.327s including waiting). Image size: 1355782243 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1f7b3957cf namespace: kserve-ci-e2e-test resourceVersion: "12694" uid: e2060955-8858-4804-9a1d-ab975712c738 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1f7ff041aa namespace: kserve-ci-e2e-test resourceVersion: "12697" uid: a22142b0-9690-4553-9edf-1d4d046d1a44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1f8066efd5 namespace: kserve-ci-e2e-test resourceVersion: "12700" uid: 08a39b60-ff73-4598-b6d7-9e4f1aaf89af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f1f80769922 namespace: kserve-ci-e2e-test resourceVersion: "12701" uid: c2982b08-ab16-4904-aef5-e1fa9afefa60 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.139s (4.139s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f2077397f3b namespace: kserve-ci-e2e-test resourceVersion: "12723" uid: 4ec82736-71ad-40ee-b31b-e51e1723b337 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f207a8dc40f namespace: kserve-ci-e2e-test resourceVersion: "12727" uid: 396427fb-fccb-4c91-9393-d0414addc36c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f207b1ef117 namespace: kserve-ci-e2e-test resourceVersion: "12729" uid: e61bb127-3a5c-47ca-9fdb-e0977b160c2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T23:21:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:22:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:37Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f20a81156ab namespace: kserve-ci-e2e-test resourceVersion: "13026" uid: 61fb33f4-5c7b-4620-8aec-0d52c9a93129 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:03Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f316421e0dd namespace: kserve-ci-e2e-test resourceVersion: "13276" uid: 9b4e5434-25bc-4ba6-8db3-34b5169c937c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:03Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f3164229f43 namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: a1f2e9ca-eefe-4bb1-a7bf-ac813362a247 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:23:08Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-09-17T23:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:08Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f329632044f namespace: kserve-ci-e2e-test resourceVersion: "13390" uid: a2aac397-0b7f-4132-a9b7-647c06cc22d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd namespace: kserve-ci-e2e-test resourceVersion: "12483" uid: d8d355e6-c535-408b-836a-266a3788443d kind: Event lastTimestamp: "2026-09-17T23:23:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: i/o timeout' metadata: creationTimestamp: "2026-09-17T23:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:08Z" name: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd.18d63f329685fd4c namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: f563a171-6e99-4f28-b52a-f50cc117c72f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b6ba0-predictor-6665558fb9 namespace: kserve-ci-e2e-test resourceVersion: "12475" uid: 4bbbb5de-32af-43d1-889d-303f904d8b3a kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'Created pod: error-404-isvc-b6ba0-predictor-6665558fb9-99hnd' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: error-404-isvc-b6ba0-predictor-6665558fb9.18d63f1b88bfe94b namespace: kserve-ci-e2e-test resourceVersion: "12482" uid: d23de719-3afd-4f77-a474-174bdb923a70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12472" uid: d9f70c04-e264-4766-9a78-beadd9d19493 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Scaled up replica set error-404-isvc-b6ba0-predictor-6665558fb9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: error-404-isvc-b6ba0-predictor.18d63f1b86fd0bd8 namespace: kserve-ci-e2e-test resourceVersion: "12476" uid: cd2c8ee5-e74d-4983-a38a-c9692729d827 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12474" uid: 32e99964-2ef6-420b-bee2-aa8d17133d12 kind: Event lastTimestamp: "2026-09-17T23:22:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:29Z" name: error-404-isvc-b6ba0-predictor.18d63f1f0563d999 namespace: kserve-ci-e2e-test resourceVersion: "12979" uid: 50f282b0-9a70-40a0-aee2-a18c3b11f937 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12474" uid: 32e99964-2ef6-420b-bee2-aa8d17133d12 kind: Event lastTimestamp: "2026-09-17T23:22:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:29Z" name: error-404-isvc-b6ba0-predictor.18d63f1f05ce10ae namespace: kserve-ci-e2e-test resourceVersion: "12980" uid: c6785d7b-0806-4818-bcb3-61fc95e17497 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:22:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12672" uid: 32e99964-2ef6-420b-bee2-aa8d17133d12 kind: Event lastTimestamp: "2026-09-17T23:22:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:59Z" name: error-404-isvc-b6ba0-predictor.18d63f26033aad51 namespace: kserve-ci-e2e-test resourceVersion: "13200" uid: d16710ad-bb99-4a2a-83bb-60f72e99c1b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:22:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12672" uid: 32e99964-2ef6-420b-bee2-aa8d17133d12 kind: Event lastTimestamp: "2026-09-17T23:22:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:59Z" name: error-404-isvc-b6ba0-predictor.18d63f2603a311b0 namespace: kserve-ci-e2e-test resourceVersion: "13201" uid: 9d46a850-8457-4f97-9c0c-537a9d842bd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6ba0 namespace: kserve-ci-e2e-test resourceVersion: "12457" uid: f6755418-3843-43e3-b988-76c499e1e69b kind: Event lastTimestamp: "2026-09-17T23:22:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:22:47Z" name: error-404-isvc-b6ba0.18d63f1b86fe9003 namespace: kserve-ci-e2e-test resourceVersion: "13094" uid: 91b41acc-34c4-4448-a9fe-9010b58298b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6ba0 namespace: kserve-ci-e2e-test resourceVersion: "13092" uid: f6755418-3843-43e3-b988-76c499e1e69b kind: Event lastTimestamp: "2026-09-17T23:22:47Z" message: InferenceService [error-404-isvc-b6ba0] is Ready metadata: creationTimestamp: "2026-09-17T23:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:22:47Z" name: error-404-isvc-b6ba0.18d63f2db818a169 namespace: kserve-ci-e2e-test resourceVersion: "13093" uid: a44d3d20-95e9-42a1-ac62-59e5803e56b5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:33:28.665493Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d6263-predictor-5f7487c565-5vmd2 to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:33:28Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc2ff489341 namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: d69d423d-41e4-4554-85ba-65b617691da9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc32166456d namespace: kserve-ci-e2e-test resourceVersion: "17086" uid: 7dccdf9e-e110-45bf-ac21-7ba7979346bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc323234986 namespace: kserve-ci-e2e-test resourceVersion: "17089" uid: 0c3a6452-a30e-4836-9e52-604085bb9813 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc325cc1171 namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: 9030196a-5b40-44de-97d9-0bbd8a184824 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc3263b4aae namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: 2583defe-a414-4c53-b0ab-94dca2ad898c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc326456fd5 namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: f474b47d-60e4-416f-8897-a87f623536bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc328fd2de4 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 6f89f18f-6ba6-43c3-aed6-ac11a57a9995 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc329680f18 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 344aaa2b-964a-4174-8d15-da5d875ecdf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T23:33:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:42:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:33:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:45Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d63fc33caecfbe namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 9570cfd6-a36c-42df-a7e9-d9cdc3176a08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:43Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d64044354f57bd namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: 40e7f8e7-4e85-4b49-8e24-92efde185e4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:43Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d6404435506c85 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: afbd9b8f-5e20-4d32-b8f5-7fc588326cfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7656f818-9d96-4f57-8334-ff021de9d720 kind: Event lastTimestamp: "2026-09-17T23:42:45Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:45Z" name: error-404-isvc-d6263-predictor-5f7487c565-5vmd2.18d64044b11cbb69 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: f951ebdf-f4ea-4c00-b641-e84edfbd064f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d6263-predictor-5f7487c565 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: e21b2f6a-458c-4caa-88d2-97a1d78904dc kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: 'Created pod: error-404-isvc-d6263-predictor-5f7487c565-5vmd2' metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:28Z" name: error-404-isvc-d6263-predictor-5f7487c565.18d63fc2fedc01c1 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: e2478aec-89db-445d-8f6e-4d4d75cabaed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: e457ad28-ee6f-4708-bc85-fa41756d0091 kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Scaled up replica set error-404-isvc-d6263-predictor-5f7487c565 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:28Z" name: error-404-isvc-d6263-predictor.18d63fc2fce81f56 namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 37451f6b-71dd-4cbf-9b8b-ee0131e562fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: 6ad6497d-f524-45b7-b6b3-67156a125f65 kind: Event lastTimestamp: "2026-09-17T23:33:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:58Z" name: error-404-isvc-d6263-predictor.18d63fc67b5ca364 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: 283aeca4-f46e-484f-9d82-ca8ae01d947e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: 6ad6497d-f524-45b7-b6b3-67156a125f65 kind: Event lastTimestamp: "2026-09-17T23:33:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:58Z" name: error-404-isvc-d6263-predictor.18d63fc67bbd5183 namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: bc55baf5-ab27-4978-b408-112d873a5b08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:34:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 6ad6497d-f524-45b7-b6b3-67156a125f65 kind: Event lastTimestamp: "2026-09-17T23:34:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:58Z" name: error-404-isvc-d6263-predictor.18d63fcd79152430 namespace: kserve-ci-e2e-test resourceVersion: "17880" uid: 4de3bea9-6c42-43f0-94ea-4743c4ca9d98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:34:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 6ad6497d-f524-45b7-b6b3-67156a125f65 kind: Event lastTimestamp: "2026-09-17T23:34:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:58Z" name: error-404-isvc-d6263-predictor.18d63fcd797d322e namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: 937c7800-55c7-4744-958c-a2e16f188923 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: b70fca65-b8d8-4ba8-9531-00d4fb86ce6e kind: Event lastTimestamp: "2026-09-17T23:34:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:34:15Z" name: error-404-isvc-d6263.18d63fc2fcec938e namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: 7566001b-87ac-40a1-bca2-3dc1241f96b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17570" uid: b70fca65-b8d8-4ba8-9531-00d4fb86ce6e kind: Event lastTimestamp: "2026-09-17T23:34:15Z" message: InferenceService [error-404-isvc-d6263] is Ready metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:34:15Z" name: error-404-isvc-d6263.18d63fcdf8872d32 namespace: kserve-ci-e2e-test resourceVersion: "17571" uid: bb74bece-f378-48a5-8878-f5d58719d41c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:44:04.058094Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d64056efa24ad1 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: ad6b363e-af6f-4a39-95cb-393d66c34ca6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d6405711ed999b namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 49c9ee7c-b104-4115-b1ae-b97b4eee4dcf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d64057135934ae namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 1eb4831b-9849-46af-baac-5ce93a3a2778 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d6405716701a09 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: ce817a46-8ecd-4d04-966b-1bd20f35b1a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d6405716e1dd05 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: d1995b0d-9992-4e08-9c0b-1ccb89e9c56a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d6405716ec4fb4 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 7caaab6a-bc20-45fd-9131-ab09e1c65c53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d6405719d875e3 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 3b22ca7d-0c79-44a3-81fc-cffb298c42b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d640571a4f0442 namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: b9745a0a-92d7-476b-b61c-599d226fff31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:44:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:41Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d640574c205488 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 4d48afd1-c554-425f-9124-8b3e1a0679c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:53:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:53:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:28Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d640da6b19d546 namespace: kserve-ci-e2e-test resourceVersion: "23866" uid: cd368522-8474-4755-b55c-129cb19efab4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:53:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: d62ed706-b889-47f5-86dc-2cf982f74298 kind: Event lastTimestamp: "2026-09-17T23:53:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:28Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l.18d640da6b1a606e namespace: kserve-ci-e2e-test resourceVersion: "23867" uid: 4ec03d29-33e5-4a08-ba92-468d680fd942 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-daa97-predictor-7b69c6c9cf namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 1c7dad0e-7722-4878-ad2e-e43978bc6537 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: 'Created pod: error-404-isvc-daa97-predictor-7b69c6c9cf-flk4l' metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor-7b69c6c9cf.18d64056ef3a14d3 namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: 2521b309-795a-40ba-ae7d-183a591c4caa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 7c8805a6-b1c8-4201-923c-a94b5da5267b kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: Scaled up replica set error-404-isvc-daa97-predictor-7b69c6c9cf from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:04Z" name: error-404-isvc-daa97-predictor.18d64056ed6da72c namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: b1470911-239b-46fe-9369-256f68ab2137 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: 1fac4221-6bf5-4223-9340-8de76485e10e kind: Event lastTimestamp: "2026-09-17T23:44:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:34Z" name: error-404-isvc-daa97-predictor.18d6405a6c993d11 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 5343e7d7-b1b7-486b-b9f8-42dcf85139fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: 1fac4221-6bf5-4223-9340-8de76485e10e kind: Event lastTimestamp: "2026-09-17T23:44:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:34Z" name: error-404-isvc-daa97-predictor.18d6405a6d089c3b namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: 0306e8e4-61c1-4351-927c-f4833a5b2193 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:44:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 1fac4221-6bf5-4223-9340-8de76485e10e kind: Event lastTimestamp: "2026-09-17T23:45:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:19Z" name: error-404-isvc-daa97-predictor.18d640616ac580bd namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 65d44838-d917-43b0-9860-131426574ee3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:44:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 1fac4221-6bf5-4223-9340-8de76485e10e kind: Event lastTimestamp: "2026-09-17T23:45:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:19Z" name: error-404-isvc-daa97-predictor.18d640616b57ee4f namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: e39ed081-087f-4baf-84ec-87a89d3ca905 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: bf96c58d-0af2-4bb9-957d-2ce284818531 kind: Event lastTimestamp: "2026-09-17T23:44:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:51Z" name: error-404-isvc-daa97.18d64056edfdeef7 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: abc9bb22-15cd-419a-a7bb-9a3c0adf9b6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: bf96c58d-0af2-4bb9-957d-2ce284818531 kind: Event lastTimestamp: "2026-09-17T23:44:51Z" message: InferenceService [error-404-isvc-daa97] is Ready metadata: creationTimestamp: "2026-09-17T23:44:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:51Z" name: error-404-isvc-daa97.18d640620cab88c2 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: 13d3970a-4fae-43a6-94a9-b92a9cd2d934 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:42:43.675917Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:42:43Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d64044387b92a3 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 03a9f4b0-ca6c-42cb-a792-22240cc5b4c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d640445a575b4d namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: f5287f1c-9860-48b6-bc8f-d03fcdb44f23 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d640445be41e92 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: def1bb46-00ec-4d7c-978f-458597fc8fb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d640445e81c92d namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: f07399cd-fd21-4dc6-ad75-900b3ade3ba6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d640445eee2ab2 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: a11bf282-0af0-4d6d-a964-018216617f83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d640445ef66809 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: c798f7b6-9365-411e-9469-9ee83feac9c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d64044617cbb84 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 5f74a1b8-e2f2-47e8-b492-4e5fcdad21ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d6404461ea8f0d namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 518b8977-3347-47f1-a676-52656309b005 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:43:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:21Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d6404495388a22 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: b1f338ff-a728-4e78-9207-fb3d5d033541 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:03Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d64056e02a5137 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: d04e2ffe-44e1-4ee9-b859-aa7ebedd9a38 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:03Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d64056e02ac965 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 253bd0a2-6621-4f57-8899-164f7cf3591a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 7c9b3305-224f-4a35-b878-14d7b23d2c19 kind: Event lastTimestamp: "2026-09-17T23:44:06Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:06Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np.18d64057717b9bf1 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: c8803286-050f-4852-a957-5a8976608efa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fc189-predictor-7d7c48b5bf namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: d011692b-02c5-465f-9e59-8da4a6bfca6f kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: 'Created pod: error-404-isvc-fc189-predictor-7d7c48b5bf-sm4np' metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:42:43Z" name: error-404-isvc-fc189-predictor-7d7c48b5bf.18d6404437ec1743 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 607fcc3d-63f5-4eeb-b8a7-021a806a1437 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 1465b88a-f8af-4a91-9931-d2812da31e98 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Scaled up replica set error-404-isvc-fc189-predictor-7d7c48b5bf from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:42:43Z" name: error-404-isvc-fc189-predictor.18d6404435bbaffd namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: c0528638-6592-4192-a18b-8c9c646b2c77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:42:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: c9997da9-523a-448e-8bba-1008a27d7008 kind: Event lastTimestamp: "2026-09-17T23:43:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:13Z" name: error-404-isvc-fc189-predictor.18d64047b485b586 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: e28ba7fc-cd11-4ab9-be1c-f90797d82da1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:42:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: c9997da9-523a-448e-8bba-1008a27d7008 kind: Event lastTimestamp: "2026-09-17T23:43:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:13Z" name: error-404-isvc-fc189-predictor.18d64047b4fa4010 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 88366a10-5bdd-47a3-b260-2e17f44beb48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:43:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: c9997da9-523a-448e-8bba-1008a27d7008 kind: Event lastTimestamp: "2026-09-17T23:43:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:43Z" name: error-404-isvc-fc189-predictor.18d6404eb255d4ad namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: d92b72e5-7d14-4839-9db7-c4a339c0085e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:43:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: c9997da9-523a-448e-8bba-1008a27d7008 kind: Event lastTimestamp: "2026-09-17T23:43:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:43Z" name: error-404-isvc-fc189-predictor.18d6404eb2bd6bf3 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: c020472b-a595-4608-89de-86d634a14d53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc189 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: b2799855-fae2-4147-901f-b6e17bef72f0 kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:43:31Z" name: error-404-isvc-fc189.18d6404435ebc0c8 namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: 86de77a0-b488-4c23-9662-dc770aa55c0b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc189 namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: b2799855-fae2-4147-901f-b6e17bef72f0 kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: InferenceService [error-404-isvc-fc189] is Ready metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:43:31Z" name: error-404-isvc-fc189.18d6404f50fa08b7 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: e9a7f952-fb3b-41b3-a47c-349de242b128 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:21:29.036571Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12401" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1b720eafa2 namespace: kserve-ci-e2e-test resourceVersion: "12405" uid: f7b3d9ac-471f-441d-ad70-908ad5b40c67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12412" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1b959ebbd6 namespace: kserve-ci-e2e-test resourceVersion: "12521" uid: 77bb88e8-87ac-41c3-a276-0a4cc1e451e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fda42133f9b1cd73575073071ee5583bebe6a773707fa25ca330879e0107d1d7" already present on machine metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1b970aef5a namespace: kserve-ci-e2e-test resourceVersion: "12525" uid: 0847a3f9-6dc6-41c4-9b83-a2ddb65795c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1b9a0b7fcf namespace: kserve-ci-e2e-test resourceVersion: "12529" uid: 73ed6d41-8ce2-4d99-b4f7-7daa0a03bc30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1b9a778ec1 namespace: kserve-ci-e2e-test resourceVersion: "12530" uid: 0a449318-fcdd-40b5-95b5-8bc134f4da8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:35Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1976" metadata: creationTimestamp: "2026-09-17T23:21:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:35Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1ce6ab2ffa namespace: kserve-ci-e2e-test resourceVersion: "12604" uid: 65e2940b-c7bf-4742-8ad9-5df0aa31683b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1976" in 11.097s (11.097s including waiting). Image size: 1580775854 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1f7c22cbfa namespace: kserve-ci-e2e-test resourceVersion: "12696" uid: d880c293-d063-42ca-bb3b-ec73185b95ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1f805204ff namespace: kserve-ci-e2e-test resourceVersion: "12699" uid: 723c8685-e5ac-42ef-88d1-6804d7e60643 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1f80d2c434 namespace: kserve-ci-e2e-test resourceVersion: "12704" uid: ca0d612d-b0d6-4ba4-b019-fd20abaa871c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f1f80dd66aa namespace: kserve-ci-e2e-test resourceVersion: "12705" uid: 0bbca286-a58f-4e6b-82c7-f387642b2081 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.088s (4.088s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f20748984d4 namespace: kserve-ci-e2e-test resourceVersion: "12721" uid: 3ef988f8-196d-450c-8291-f1667b294290 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f2079394ff3 namespace: kserve-ci-e2e-test resourceVersion: "12724" uid: b32a138e-8427-4af6-9bee-bf486689132c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f2079f212fe namespace: kserve-ci-e2e-test resourceVersion: "12725" uid: 706e2b68-a581-43a2-940d-f9800d2b9c9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-17T23:21:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:23:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:21:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:53Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f20e44ae4eb namespace: kserve-ci-e2e-test resourceVersion: "13888" uid: 4c8307e7-a247-4905-9248-c7c4c324e82e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f3c3701cec4 namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: 8b243c2a-9892-4d28-9cd2-0c01bec7d42e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f3c37023b28 namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 326f8b67-8d6f-4854-9574-5027ede2681c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8 namespace: kserve-ci-e2e-test resourceVersion: "12403" uid: 98cd3953-a341-4906-97cc-0f9a96a74678 kind: Event lastTimestamp: "2026-09-17T23:23:53Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:53Z" name: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8.18d63f3d10c1e6b8 namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: 03df375c-af1c-453a-8a75-9ceece8db379 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-786b497f89 namespace: kserve-ci-e2e-test resourceVersion: "12393" uid: c1685817-3268-4c92-a21c-217d30d20674 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-786b497f89-ntlt8' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-1-predictor-786b497f89.18d63f1b718220fa namespace: kserve-ci-e2e-test resourceVersion: "12402" uid: e501275f-4ce8-43a2-b348-6b2a3270c23c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12391" uid: 285d621b-7e54-411c-a22a-50d22feb3239 kind: Event lastTimestamp: "2026-09-17T23:21:28Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-786b497f89 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:28Z" name: isvc-sklearn-graph-1-predictor.18d63f1b6cccee43 namespace: kserve-ci-e2e-test resourceVersion: "12394" uid: dde1ef6a-8524-4683-8cf9-90f8fdfc4604 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12406" uid: cd5fd1ad-a21c-49ec-8ebe-529a1247caec kind: Event lastTimestamp: "2026-09-17T23:22:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:29Z" name: isvc-sklearn-graph-1-predictor.18d63f1ef232d74d namespace: kserve-ci-e2e-test resourceVersion: "12963" uid: a7883bb0-2b7f-4aef-a716-768805336e3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12406" uid: cd5fd1ad-a21c-49ec-8ebe-529a1247caec kind: Event lastTimestamp: "2026-09-17T23:22:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:29Z" name: isvc-sklearn-graph-1-predictor.18d63f1ef2a0595e namespace: kserve-ci-e2e-test resourceVersion: "12964" uid: c6432679-1f3c-46d3-b747-b7595fcfa602 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:22:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12663" uid: cd5fd1ad-a21c-49ec-8ebe-529a1247caec kind: Event lastTimestamp: "2026-09-17T23:23:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:44Z" name: isvc-sklearn-graph-1-predictor.18d63f25f0d84ca8 namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: bd48584e-da7f-4515-a199-cc9571e95978 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:22:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12663" uid: cd5fd1ad-a21c-49ec-8ebe-529a1247caec kind: Event lastTimestamp: "2026-09-17T23:23:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:44Z" name: isvc-sklearn-graph-1-predictor.18d63f25f14a152d namespace: kserve-ci-e2e-test resourceVersion: "13654" uid: 2900a160-7cb8-4f3e-b2ec-54aa6858aaec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "12377" uid: 9fdaa2d2-98ad-46bd-9be1-80dc7d514ab1 kind: Event lastTimestamp: "2026-09-17T23:23:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:13Z" name: isvc-sklearn-graph-1.18d63f1b72dde5c9 namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: 1d029f46-6881-4386-bb38-d3492f90b5be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13434" uid: 9fdaa2d2-98ad-46bd-9be1-80dc7d514ab1 kind: Event lastTimestamp: "2026-09-17T23:23:13Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-17T23:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:13Z" name: isvc-sklearn-graph-1.18d63f33d2673aaf namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: 34e39219-add4-4268-bdda-8914a47e0d5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:21:29.609859Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12509" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-867659858b-x295x to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1b943a58b9 namespace: kserve-ci-e2e-test resourceVersion: "12515" uid: ed3c0d63-f712-4d24-9e8b-e017cdc0a38a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1ba501a7e1 namespace: kserve-ci-e2e-test resourceVersion: "12538" uid: f2c63bc2-3525-418a-b048-f54ff5f9db0c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12522" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:21:30Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1bdc03397a namespace: kserve-ci-e2e-test resourceVersion: "12565" uid: 654e0fee-93e7-4fb2-9a7f-650489d5cf69 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fda42133f9b1cd73575073071ee5583bebe6a773707fa25ca330879e0107d1d7" already present on machine metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1bddc3be32 namespace: kserve-ci-e2e-test resourceVersion: "12567" uid: 9bec397b-4cd8-404f-8eb5-ac63bd75526d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1be232d7d4 namespace: kserve-ci-e2e-test resourceVersion: "12568" uid: 2f9dc165-3c6a-49eb-b387-200bc9c18e8c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1be37c2d47 namespace: kserve-ci-e2e-test resourceVersion: "12569" uid: 2c75e47c-76d1-4e38-a201-4e424daaa08e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:36Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1976" metadata: creationTimestamp: "2026-09-17T23:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:36Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1d22bfa4f3 namespace: kserve-ci-e2e-test resourceVersion: "12611" uid: e51416fb-bb5f-461d-b6d9-1e0bf7005c41 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1976" in 10.08s (10.08s including waiting). Image size: 1580775854 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1f7b9249a2 namespace: kserve-ci-e2e-test resourceVersion: "12695" uid: 1af15359-03e3-4a17-b4e6-8837c06dfbbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1f804305a0 namespace: kserve-ci-e2e-test resourceVersion: "12698" uid: cf29f8a0-420c-43d8-a15d-58997df30549 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1f80c610c4 namespace: kserve-ci-e2e-test resourceVersion: "12702" uid: 467ee8e3-70f3-4606-9045-8b7a8f0ad4ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:46Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T23:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:46Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f1f80d2706f namespace: kserve-ci-e2e-test resourceVersion: "12703" uid: c8fd6f25-6289-4346-b39d-6c89389c8ace reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.103s (4.103s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f207562eefc namespace: kserve-ci-e2e-test resourceVersion: "12722" uid: cd9156bb-8d73-412e-af91-8f1512ed58a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f207a8019f7 namespace: kserve-ci-e2e-test resourceVersion: "12726" uid: f31e72f0-8ff8-4322-8481-8527649c525c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:21:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:50Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f207b0726e2 namespace: kserve-ci-e2e-test resourceVersion: "12728" uid: bc49ab57-0f9d-4339-adad-cf40346f387c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T23:21:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:22:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:21:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:58Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f20e44a5f66 namespace: kserve-ci-e2e-test resourceVersion: "13188" uid: 7c67b50b-a512-4027-942f-a81d96db17e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f3c3ed8e691 namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 6f7c5547-5943-4bea-94e5-3f74b8fd5a84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f3c3edf307c namespace: kserve-ci-e2e-test resourceVersion: "13769" uid: 62e6bc11-f141-4f7a-b058-a69cf344dc27 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-867659858b-x295x namespace: kserve-ci-e2e-test resourceVersion: "12513" uid: ae705b50-d86a-4638-929a-7bb5b7315285 kind: Event lastTimestamp: "2026-09-17T23:23:53Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:53Z" name: isvc-sklearn-graph-2-predictor-867659858b-x295x.18d63f3d10c1607c namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: 949e0b61-f724-4ccf-a823-b18300e0ede4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-867659858b namespace: kserve-ci-e2e-test resourceVersion: "12502" uid: 51b68499-10e6-411c-9021-02e2dc90404a kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-867659858b-x295x' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-2-predictor-867659858b.18d63f1b93a831c8 namespace: kserve-ci-e2e-test resourceVersion: "12512" uid: ff9382f8-8f84-42c3-8e45-5cc022c8e76d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12500" uid: 6206dbaf-8d5e-4ab3-b7f8-ce83d11afcdc kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-867659858b from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: isvc-sklearn-graph-2-predictor.18d63f1b90a44dd5 namespace: kserve-ci-e2e-test resourceVersion: "12503" uid: 795a50a5-fbd2-4f67-a02b-ad6b3c2afab3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12501" uid: 4e7cd84c-25e8-4cb2-8250-c872b24b68e3 kind: Event lastTimestamp: "2026-09-17T23:22:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:29Z" name: isvc-sklearn-graph-2-predictor.18d63f1f0f199d34 namespace: kserve-ci-e2e-test resourceVersion: "12982" uid: ce24fe7d-02ee-47cd-8a46-d844d2423570 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12501" uid: 4e7cd84c-25e8-4cb2-8250-c872b24b68e3 kind: Event lastTimestamp: "2026-09-17T23:22:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:29Z" name: isvc-sklearn-graph-2-predictor.18d63f1f0f774722 namespace: kserve-ci-e2e-test resourceVersion: "12983" uid: 90f210c8-bb09-4735-a62c-8811fac11ec1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12674" uid: 4e7cd84c-25e8-4cb2-8250-c872b24b68e3 kind: Event lastTimestamp: "2026-09-17T23:23:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:29Z" name: isvc-sklearn-graph-2-predictor.18d63f2d0a784d2d namespace: kserve-ci-e2e-test resourceVersion: "13538" uid: 04a559d1-a00a-4bc0-8584-5ad0f9f213cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:22:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "12674" uid: 4e7cd84c-25e8-4cb2-8250-c872b24b68e3 kind: Event lastTimestamp: "2026-09-17T23:23:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:29Z" name: isvc-sklearn-graph-2-predictor.18d63f2d0aea40bf namespace: kserve-ci-e2e-test resourceVersion: "13539" uid: eaa958e3-2915-4bb1-a209-03c95ddd40f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "12492" uid: 9838a6c3-2732-4c61-8869-a93c86ec0cfd kind: Event lastTimestamp: "2026-09-17T23:23:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:08Z" name: isvc-sklearn-graph-2.18d63f1b90a5cdd7 namespace: kserve-ci-e2e-test resourceVersion: "13400" uid: 345978da-f020-4e4f-8419-bb6787fe92e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13398" uid: 9838a6c3-2732-4c61-8869-a93c86ec0cfd kind: Event lastTimestamp: "2026-09-17T23:23:08Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-17T23:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:08Z" name: isvc-sklearn-graph-2.18d63f329c1e9cb2 namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: 318e4575-9f92-4449-8446-b3e9652b92f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:21:29.336263Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12458" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-xjn72 to ip-10-0-128-70.ec2.internal metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:21:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1b83eb844a namespace: kserve-ci-e2e-test resourceVersion: "12463" uid: ad81037b-a47a-4f49-bc9e-defea7cf734e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1b92e3d064 namespace: kserve-ci-e2e-test resourceVersion: "12508" uid: 3732e1df-3a66-45b4-b1bd-c57e098d81fc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12473" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:21:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1bca3b4b73 namespace: kserve-ci-e2e-test resourceVersion: "12558" uid: a93c291f-b25d-4c93-a3f7-c27e9be937d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:fda42133f9b1cd73575073071ee5583bebe6a773707fa25ca330879e0107d1d7" already present on machine metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1bcbb5f971 namespace: kserve-ci-e2e-test resourceVersion: "12560" uid: 6077a166-7a6c-4850-82d0-db187e45f1e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1bd0ed79d4 namespace: kserve-ci-e2e-test resourceVersion: "12561" uid: a1d16fcd-9d4c-4626-88cd-039b8343666d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1bd2196243 namespace: kserve-ci-e2e-test resourceVersion: "12563" uid: f4293538-3dba-4683-b8e8-327274235f43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:21:36Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-17T23:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f1d22de6f4b namespace: kserve-ci-e2e-test resourceVersion: "12612" uid: c95f1745-e3b8-4f3c-895a-fb7aac26b848 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:22:07Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 30.717s (30.717s including waiting). Image size: 1435742980 bytes.' metadata: creationTimestamp: "2026-09-17T23:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f2449ce04df namespace: kserve-ci-e2e-test resourceVersion: "12834" uid: d0f0ac04-2125-4c46-a01f-67745d40d9ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:22:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f244d58cd42 namespace: kserve-ci-e2e-test resourceVersion: "12835" uid: 72da50fb-42cf-42ad-8091-ad8c7d77da8d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:22:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f244e0b0803 namespace: kserve-ci-e2e-test resourceVersion: "12836" uid: 41b8d658-c1c9-4381-bf12-82225fb63f03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:22:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f244e2890b1 namespace: kserve-ci-e2e-test resourceVersion: "12837" uid: 252a31e5-a51b-4b58-b25d-fbbacc8189ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:22:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f2450faafa9 namespace: kserve-ci-e2e-test resourceVersion: "12839" uid: b87fdd40-1125-4ab0-850c-3fbdf2aab834 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:22:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f245158c3a5 namespace: kserve-ci-e2e-test resourceVersion: "12840" uid: f44740f2-2a6a-479a-b19a-a3f14a429dc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T23:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:23:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f24a3147109 namespace: kserve-ci-e2e-test resourceVersion: "13444" uid: 6e9c6778-ac47-41ef-bacd-7b33302610ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f3c3c65b352 namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: dc6af6c5-e234-4c2f-8e31-6ffe96a8e072 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xjn72 namespace: kserve-ci-e2e-test resourceVersion: "12461" uid: fec938fb-b703-47d1-b349-0b4022eacb08 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-xjn72.18d63f3c3c664c22 namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: 13d2bf6c-aa6b-4561-a5d9-d8344b049d6d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-70.ec2.internal source: component: kubelet host: ip-10-0-128-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "12448" uid: 2f97d738-66b2-4551-aad7-f7d2a103e6cb kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-xjn72' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d63f1b83356217 namespace: kserve-ci-e2e-test resourceVersion: "12460" uid: 738ec19b-845b-4ebf-b596-ac844a22d6da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12446" uid: adeb6f97-d02a-4b4f-a758-ded86265bd27 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: isvc-xgboost-graph-predictor.18d63f1b7fceb6f2 namespace: kserve-ci-e2e-test resourceVersion: "12450" uid: cd6e08fd-7926-4cfd-89d9-9c8dd4aa2b00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12447" uid: 1e453fcc-956f-4e9b-8a7c-4de442ac9de8 kind: Event lastTimestamp: "2026-09-17T23:22:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:14Z" name: isvc-xgboost-graph-predictor.18d63f1efe52cf4d namespace: kserve-ci-e2e-test resourceVersion: "12881" uid: 7386bf2c-37c6-4525-9d45-52037b9cba6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12447" uid: 1e453fcc-956f-4e9b-8a7c-4de442ac9de8 kind: Event lastTimestamp: "2026-09-17T23:22:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:14Z" name: isvc-xgboost-graph-predictor.18d63f1efec3ee46 namespace: kserve-ci-e2e-test resourceVersion: "12882" uid: 941375fd-9399-4a0c-8fae-09870f669f3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:22:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12669" uid: 1e453fcc-956f-4e9b-8a7c-4de442ac9de8 kind: Event lastTimestamp: "2026-09-17T23:23:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:44Z" name: isvc-xgboost-graph-predictor.18d63f297aeb0994 namespace: kserve-ci-e2e-test resourceVersion: "13655" uid: 552ee1cc-2a5b-4110-8d21-fdd3e71f9464 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:22:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "12669" uid: 1e453fcc-956f-4e9b-8a7c-4de442ac9de8 kind: Event lastTimestamp: "2026-09-17T23:23:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:44Z" name: isvc-xgboost-graph-predictor.18d63f297b64af7a namespace: kserve-ci-e2e-test resourceVersion: "13656" uid: e3d524b9-0a9d-4b8e-9019-3be3328f5b80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "12430" uid: d65166b9-ce56-4734-95f4-af16c5da4619 kind: Event lastTimestamp: "2026-09-17T23:23:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:24Z" name: isvc-xgboost-graph.18d63f1b7fe05f6b namespace: kserve-ci-e2e-test resourceVersion: "13514" uid: f5d921f8-a2c0-450a-b7e0-8ab9b00abe1d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: d65166b9-ce56-4734-95f4-af16c5da4619 kind: Event lastTimestamp: "2026-09-17T23:23:24Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-17T23:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:24Z" name: isvc-xgboost-graph.18d63f365a095339 namespace: kserve-ci-e2e-test resourceVersion: "13513" uid: b23bf8e8-3563-4e2e-939a-7a1ec53d3d02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:23:38.979757Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5c78d4956c-gjrh7 to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:23:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:23:38Z" name: model-chainer-5c78d4956c-gjrh7.18d63f39b34657b8 namespace: kserve-ci-e2e-test resourceVersion: "13601" uid: 06b8eeb0-c054-4726-8484-5126d4097c99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13605" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: "2026-09-17T23:23:39Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:23:39Z" name: model-chainer-5c78d4956c-gjrh7.18d63f39d536b10b namespace: kserve-ci-e2e-test resourceVersion: "13618" uid: c1d504a9-34f7-468e-b552-fca95e17ab81 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: "2026-09-17T23:23:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:39Z" name: model-chainer-5c78d4956c-gjrh7.18d63f39d67f0f5b namespace: kserve-ci-e2e-test resourceVersion: "13620" uid: c3840fcd-e1ba-4395-b786-944b4206c116 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: "2026-09-17T23:23:39Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-17T23:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:39Z" name: model-chainer-5c78d4956c-gjrh7.18d63f39d958a951 namespace: kserve-ci-e2e-test resourceVersion: "13621" uid: e9517ec6-7cf1-4eda-9d70-54cb9168300f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: "2026-09-17T23:23:39Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-17T23:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:39Z" name: model-chainer-5c78d4956c-gjrh7.18d63f39d9db81c9 namespace: kserve-ci-e2e-test resourceVersion: "13622" uid: 71f524dd-90bc-4d4b-82d9-455151ee65da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: "2026-09-17T23:23:49Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-17T23:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:49Z" name: model-chainer-5c78d4956c-gjrh7.18d63f3c2d6ce19d namespace: kserve-ci-e2e-test resourceVersion: "13711" uid: c4ae8cff-93b9-4b9b-bdd4-8461df873760 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5c78d4956c-gjrh7 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 3365ee42-d2b4-4fd9-a435-d03b7ebdd036 kind: Event lastTimestamp: "2026-09-17T23:24:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:16Z" name: model-chainer-5c78d4956c-gjrh7.18d63f3c9cb6de55 namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: 388b06ce-dc28-4d63-9a7a-0a5ca5b1f56d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5c78d4956c namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: 526f70a4-27c3-4889-b0b3-6119df266bad kind: Event lastTimestamp: "2026-09-17T23:23:38Z" message: 'Created pod: model-chainer-5c78d4956c-gjrh7' metadata: creationTimestamp: "2026-09-17T23:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:38Z" name: model-chainer-5c78d4956c.18d63f39b291d625 namespace: kserve-ci-e2e-test resourceVersion: "13599" uid: 571f8c23-2b78-474f-9961-5b9f15e85de2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "13591" uid: 8fa07751-aa06-4c88-bced-320e22b9abe2 kind: Event lastTimestamp: "2026-09-17T23:23:38Z" message: Scaled up replica set model-chainer-5c78d4956c from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:38Z" name: model-chainer.18d63f39b13c76b0 namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: 9077ea09-6b0d-47e8-9095-846590846e36 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "13607" uid: 0b435323-4ea7-4bb6-8f30-200dcab2450c kind: Event lastTimestamp: "2026-09-17T23:23:39Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-17T23:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:39Z" name: model-chainer.18d63f39b7b25434 namespace: kserve-ci-e2e-test resourceVersion: "13609" uid: 687e32d1-2084-434f-8093-1cb11cab0f7e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:24:49.838294Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2026c-6498f87569-5x5bj to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:24:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:24:49Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63f4a32c5b8d5 namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: 97c0646b-83fb-49f2-8fd6-0668740c3523 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: "2026-09-17T23:24:50Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:24:50Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63f4a54c6cd10 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: 46d2d581-913b-49ad-bd00-96e1baa1cc33 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2026c} kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: "2026-09-17T23:24:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:50Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63f4a55fb1c39 namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: d512cc52-61a7-4ae7-9edd-4f0d3f5c66ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2026c} kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: "2026-09-17T23:24:50Z" message: 'Created container: sequence-graph-2026c' metadata: creationTimestamp: "2026-09-17T23:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:50Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63f4a58caada9 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: dea637a7-4d4b-42da-b611-1f2cb9cb797f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2026c} kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: "2026-09-17T23:24:50Z" message: Started container sequence-graph-2026c metadata: creationTimestamp: "2026-09-17T23:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:50Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63f4a594ce71f namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 6f3855a1-5e2d-478d-ba3a-4ad38cede92e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2026c} kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Stopping container sequence-graph-2026c metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:04Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63fbd5c6a4fa5 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 3f76b8fb-0279-4f32-a3db-87e54b4f44d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2026c} kind: Pod name: sequence-graph-2026c-6498f87569-5x5bj namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 5f608d0b-e29e-4f6f-b8e0-edb51c43fa7e kind: Event lastTimestamp: "2026-09-17T23:33:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:31Z" name: sequence-graph-2026c-6498f87569-5x5bj.18d63fbde817e322 namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: 3d690585-eff3-4d6a-bd55-f59427653b75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2026c-6498f87569 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: 289d5662-c166-4a8b-9980-c11c21d7f81d kind: Event lastTimestamp: "2026-09-17T23:24:49Z" message: 'Created pod: sequence-graph-2026c-6498f87569-5x5bj' metadata: creationTimestamp: "2026-09-17T23:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:49Z" name: sequence-graph-2026c-6498f87569.18d63f4a323cfb1c namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: d435ab6c-6d2b-4931-826c-2c009b07b874 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2026c namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 62b86ca8-d299-4cdc-8d38-001997596110 kind: Event lastTimestamp: "2026-09-17T23:24:49Z" message: Scaled up replica set sequence-graph-2026c-6498f87569 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:49Z" name: sequence-graph-2026c.18d63f4a3098270a namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: d8d9dac9-5e68-4f1d-b898-bb9d4c3ee241 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2026c namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: c08d67e9-f5e1-4ef1-999d-33bcbfbef26a kind: Event lastTimestamp: "2026-09-17T23:24:49Z" message: InferenceGraph [sequence-graph-2026c] is Ready metadata: creationTimestamp: "2026-09-17T23:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:49Z" name: sequence-graph-2026c.18d63f4a39428151 namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: c40c9f9b-c615-4600-ab08-62621bfe1877 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:25:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2026c namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: d133b0da-b965-404f-89e1-9ac7669f7097 kind: Event lastTimestamp: "2026-09-17T23:25:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:25:34Z" name: sequence-graph-2026c.18d63f4daf308533 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 7e6e9735-131a-4d8a-b648-5b3fde9f4de6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:25:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2026c namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: d133b0da-b965-404f-89e1-9ac7669f7097 kind: Event lastTimestamp: "2026-09-17T23:25:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:25:34Z" name: sequence-graph-2026c.18d63f4daf95a543 namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: ad5f835c-0055-4396-9518-ff036cec14a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:34:04.677848Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-4a1f9-5868fb9865-dwz95 to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:34:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:34:04Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcb61c97954 namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: d15be16c-2b44-4e7b-bcc1-a99ef96a7063 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:34:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-4a1f9-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:05Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcb6a896e98 namespace: kserve-ci-e2e-test resourceVersion: "17376" uid: 7a848564-2272-406f-8cba-a892bb4e7cc6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17361" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:06Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:34:06Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcbdd1bde6f namespace: kserve-ci-e2e-test resourceVersion: "17389" uid: af8052f1-dea3-465c-b2f1-64416872f32c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4a1f9} kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:06Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcbde741e3a namespace: kserve-ci-e2e-test resourceVersion: "17391" uid: a2a3e673-744b-47dc-a088-e0e20347637a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4a1f9} kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:06Z" message: 'Created container: sequence-graph-4a1f9' metadata: creationTimestamp: "2026-09-17T23:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:06Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcbe115cae2 namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: 263e842e-b377-437f-b26a-1f265bf06911 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4a1f9} kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:06Z" message: Started container sequence-graph-4a1f9 metadata: creationTimestamp: "2026-09-17T23:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:06Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcbe18218b8 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 7974566c-2852-430d-b8a5-953687686c8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4a1f9} kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: Stopping container sequence-graph-4a1f9 metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:14Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fcdbab24e45 namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: 416fa52c-2b62-43c1-90bb-729ebceda8f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:34:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4a1f9} kind: Pod name: sequence-graph-4a1f9-5868fb9865-dwz95 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 2fb527d6-dff9-4e49-b3ff-242bd1db8fb8 kind: Event lastTimestamp: "2026-09-17T23:34:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:34:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:43Z" name: sequence-graph-4a1f9-5868fb9865-dwz95.18d63fce932f5f4c namespace: kserve-ci-e2e-test resourceVersion: "17794" uid: 4338af17-e5c7-42c0-8b62-71e72040517f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-4a1f9-5868fb9865 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 7770757c-a883-4af4-8e2d-fe3a7835e26a kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: 'Created pod: sequence-graph-4a1f9-5868fb9865-dwz95' metadata: creationTimestamp: "2026-09-17T23:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:04Z" name: sequence-graph-4a1f9-5868fb9865.18d63fcb612f2f21 namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 4781e6d1-2a1b-4105-92c3-039f43498ec1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-4a1f9 namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: 3f1ffb30-f718-4b28-8e88-e3938426ea56 kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: Scaled up replica set sequence-graph-4a1f9-5868fb9865 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:04Z" name: sequence-graph-4a1f9.18d63fcb60312297 namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: 28648bd8-42cc-462f-976d-ed559f8af393 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-4a1f9 namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 73bebc11-3f6e-4666-b84e-9a81af90a9a6 kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: InferenceGraph [sequence-graph-4a1f9] is Ready metadata: creationTimestamp: "2026-09-17T23:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:34:04Z" name: sequence-graph-4a1f9.18d63fcb6616f6d7 namespace: kserve-ci-e2e-test resourceVersion: "17369" uid: 390f6cf2-0993-48a8-b860-921139d69e2d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:35:15.024726Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17985" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-51963-6984fdf9fc-8m9mp to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:35:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:35:15Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d63fdbc2c9a7bf namespace: kserve-ci-e2e-test resourceVersion: "17991" uid: bf1da0e6-4e2f-4563-b6e4-a1877b7f7f5a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:35:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-51963-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:35:15Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d63fdbcea8a7c9 namespace: kserve-ci-e2e-test resourceVersion: "18003" uid: dcd8118a-972a-42ef-a60d-0fb7de32b523 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17994" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:35:16Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:35:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:35:16Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d63fdc08d2e1b9 namespace: kserve-ci-e2e-test resourceVersion: "18014" uid: af501e39-5c8b-4199-a040-676c26e3f699 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-51963} kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:35:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:35:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:35:16Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d63fdc0a1c792b namespace: kserve-ci-e2e-test resourceVersion: "18016" uid: 3daa3be9-d4e4-44ad-a6e1-6915cc1e5c5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-51963} kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:35:16Z" message: 'Created container: sequence-graph-51963' metadata: creationTimestamp: "2026-09-17T23:35:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:35:16Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d63fdc0ce1a231 namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: a5d1e49b-2e5b-4b23-a69a-31f47f6938d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-51963} kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:35:16Z" message: Started container sequence-graph-51963 metadata: creationTimestamp: "2026-09-17T23:35:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:35:16Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d63fdc0d661b83 namespace: kserve-ci-e2e-test resourceVersion: "18018" uid: 4a014699-6970-41a0-ac36-1db15fa928bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-51963} kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: Stopping container sequence-graph-51963 metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:29Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d6404eef808592 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 8d018ad3-cbb0-4aa7-9717-c54f1faad5c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-51963} kind: Pod name: sequence-graph-51963-6984fdf9fc-8m9mp namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 591bb5b4-95ed-46d5-aaa3-1489097a0e90 kind: Event lastTimestamp: "2026-09-17T23:43:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:57Z" name: sequence-graph-51963-6984fdf9fc-8m9mp.18d6404fa4526069 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 2d298cf9-2a6f-4306-9a11-e832ab07400a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-51963-6984fdf9fc namespace: kserve-ci-e2e-test resourceVersion: "17981" uid: 021e0d95-38eb-4d74-a94e-baf7a45be7bd kind: Event lastTimestamp: "2026-09-17T23:35:15Z" message: 'Created pod: sequence-graph-51963-6984fdf9fc-8m9mp' metadata: creationTimestamp: "2026-09-17T23:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:15Z" name: sequence-graph-51963-6984fdf9fc.18d63fdbc221689b namespace: kserve-ci-e2e-test resourceVersion: "17988" uid: d709e2d8-f524-4581-9ed6-845f8bac2079 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-51963 namespace: kserve-ci-e2e-test resourceVersion: "17979" uid: 1450673a-98c6-4e3b-82fa-777313631d47 kind: Event lastTimestamp: "2026-09-17T23:35:14Z" message: Scaled up replica set sequence-graph-51963-6984fdf9fc from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:14Z" name: sequence-graph-51963.18d63fdbc0946294 namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: ce7210e5-34a7-4974-b9d6-77b20d2b00f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-51963 namespace: kserve-ci-e2e-test resourceVersion: "17997" uid: 20841e4f-d8fd-4cff-9cca-aed28a9c4035 kind: Event lastTimestamp: "2026-09-17T23:35:15Z" message: InferenceGraph [sequence-graph-51963] is Ready metadata: creationTimestamp: "2026-09-17T23:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:35:15Z" name: sequence-graph-51963.18d63fdbc69caad1 namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: be4a0f6a-2824-4205-b1e2-46eb95fcd57b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-51963 namespace: kserve-ci-e2e-test resourceVersion: "17980" uid: db822a46-505b-4150-aa42-a51a2dc10a1a kind: Event lastTimestamp: "2026-09-17T23:35:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:35:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:29Z" name: sequence-graph-51963.18d63fdf3f0811f5 namespace: kserve-ci-e2e-test resourceVersion: "18090" uid: 17469bd3-1db2-4ca4-91d1-ae10b5f88dea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-51963 namespace: kserve-ci-e2e-test resourceVersion: "17980" uid: db822a46-505b-4150-aa42-a51a2dc10a1a kind: Event lastTimestamp: "2026-09-17T23:35:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:35:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:30Z" name: sequence-graph-51963.18d63fdf3fb81764 namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: 86e289f0-fcb6-4b53-901c-54516994d7a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-51963 namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: db822a46-505b-4150-aa42-a51a2dc10a1a kind: Event lastTimestamp: "2026-09-17T23:35:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:45Z" name: sequence-graph-51963.18d63fe2be91c322 namespace: kserve-ci-e2e-test resourceVersion: "18159" uid: 2d222d46-ae17-4869-b4e0-ade3f9c46838 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-51963 namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: db822a46-505b-4150-aa42-a51a2dc10a1a kind: Event lastTimestamp: "2026-09-17T23:35:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:45Z" name: sequence-graph-51963.18d63fe2beffa43f namespace: kserve-ci-e2e-test resourceVersion: "18161" uid: 63f59076-3479-4575-a69e-50e466ffed44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:45:13.906525Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-daa97-dfb866868-gpr4r to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:45:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:45:13Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d6406732ecbd53 namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: 024ad252-9b60-4bac-adff-e61e7f159c3e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:14Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:45:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-daa97-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:45:14Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d640673cf648ff namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 5af709cc-bacc-403b-bdd3-6c38f1451476 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:15Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:45:15Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:45:15Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d6406778f5dacd namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: c3020c80-f2ac-4f74-a818-89300cb76e7a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-daa97} kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:45:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:45:15Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d640677a5027b1 namespace: kserve-ci-e2e-test resourceVersion: "21653" uid: 04280c90-d335-4c27-a7b8-0861bb759194 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-daa97} kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:45:15Z" message: 'Created container: splitter-graph-daa97' metadata: creationTimestamp: "2026-09-17T23:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:45:15Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d640677df7e567 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: b73baac0-1259-4869-83bd-208d47dda163 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-daa97} kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:45:15Z" message: Started container splitter-graph-daa97 metadata: creationTimestamp: "2026-09-17T23:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:45:15Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d640677e7a6eec namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 80eef1dc-4e5f-4695-834e-a6dc17226c86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:53:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-daa97} kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:53:28Z" message: Stopping container splitter-graph-daa97 metadata: creationTimestamp: "2026-09-17T23:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:28Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d640da5de34753 namespace: kserve-ci-e2e-test resourceVersion: "23834" uid: f63076d2-af76-4343-ab19-bc8ea0d2c99b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-daa97} kind: Pod name: splitter-graph-daa97-dfb866868-gpr4r namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 3be8634a-66b8-4a7b-b457-394c532d9e71 kind: Event lastTimestamp: "2026-09-17T23:53:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:56Z" name: splitter-graph-daa97-dfb866868-gpr4r.18d640db09805448 namespace: kserve-ci-e2e-test resourceVersion: "23999" uid: 721dcc90-6a3e-4fd6-9951-b0b1e1a4000c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-daa97-dfb866868 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 334927ff-79e2-4107-bcd7-e4d6cbe7e75c kind: Event lastTimestamp: "2026-09-17T23:45:13Z" message: 'Created pod: splitter-graph-daa97-dfb866868-gpr4r' metadata: creationTimestamp: "2026-09-17T23:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:13Z" name: splitter-graph-daa97-dfb866868.18d640673284e0f2 namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 72bb475b-5f90-4fee-9bdc-6112545e3f92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 654d23b9-efd0-472e-ae50-21f82ccb1d58 kind: Event lastTimestamp: "2026-09-17T23:45:13Z" message: Scaled up replica set splitter-graph-daa97-dfb866868 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:13Z" name: splitter-graph-daa97.18d640673192e669 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: 64bc2b26-d4a1-459c-9460-ac83933b440e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 32366029-3208-444b-8e4e-c2d0c5930464 kind: Event lastTimestamp: "2026-09-17T23:45:13Z" message: InferenceGraph [splitter-graph-daa97] is Ready metadata: creationTimestamp: "2026-09-17T23:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:45:13Z" name: splitter-graph-daa97.18d64067378a5403 namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 33197bfa-61a0-467d-ab94-dcdf19457ca6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 4b51362d-9f6a-4201-8e0f-c2e2e1d32b56 kind: Event lastTimestamp: "2026-09-17T23:45:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:28Z" name: splitter-graph-daa97.18d6406ab06ff09f namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 422921fb-692a-4144-9895-4daa9bc400ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 4b51362d-9f6a-4201-8e0f-c2e2e1d32b56 kind: Event lastTimestamp: "2026-09-17T23:45:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:28Z" name: splitter-graph-daa97.18d6406ab0de4d5f namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 2e4855fb-ebaf-4239-8189-e577b7431f33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 4b51362d-9f6a-4201-8e0f-c2e2e1d32b56 kind: Event lastTimestamp: "2026-09-17T23:45:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:43Z" name: splitter-graph-daa97.18d6406e2fae9dbf namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: f187d95f-5352-4645-9f28-79ff8a4e3f4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 4b51362d-9f6a-4201-8e0f-c2e2e1d32b56 kind: Event lastTimestamp: "2026-09-17T23:45:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:43Z" name: splitter-graph-daa97.18d6406e3007ee51 namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 2c748fdf-e175-42ed-8e67-cb8d7e352953 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:43:53.576654Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-fc189-8457d9b884-rsm8p to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:43:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:43:53Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d640547ee422e3 namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 7ef0b634-cad2-4681-a255-39e5051b5f80 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:43:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-fc189-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:53Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d6405487eb0013 namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 2aa9922b-7afb-4ef9-94bc-00cbbe06ed8f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:54Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:43:54Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:43:54Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d64054c523361d namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: b1229cbe-cb16-43aa-8952-a87cb918b4ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fc189} kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:43:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:54Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d64054c6fb925e namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: 7391963f-6eba-49cb-b94a-471ffa217be6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fc189} kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:43:54Z" message: 'Created container: splitter-graph-fc189' metadata: creationTimestamp: "2026-09-17T23:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:54Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d64054cae73d62 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 9feafd72-696a-4805-9fbe-a8cbb53e2632 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fc189} kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:43:54Z" message: Started container splitter-graph-fc189 metadata: creationTimestamp: "2026-09-17T23:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:54Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d64054cb6fbeb8 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: 091f9a6b-67a8-48e4-896e-4e7f6fe200e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fc189} kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: Stopping container splitter-graph-fc189 metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:03Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d64056d71ee301 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: a9afaa17-d9eb-4dfe-8c8e-391331fc2309 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fc189} kind: Pod name: splitter-graph-fc189-8457d9b884-rsm8p namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 450701f9-bf58-4704-8dbc-2cf4c3274efa kind: Event lastTimestamp: "2026-09-17T23:44:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:31Z" name: splitter-graph-fc189-8457d9b884-rsm8p.18d6405765329654 namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: c1dc093e-4fab-402c-a997-f1c0c2ef9613 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-fc189-8457d9b884 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 7efd92f1-a020-4796-bf87-9692035209eb kind: Event lastTimestamp: "2026-09-17T23:43:53Z" message: 'Created pod: splitter-graph-fc189-8457d9b884-rsm8p' metadata: creationTimestamp: "2026-09-17T23:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:53Z" name: splitter-graph-fc189-8457d9b884.18d640547de71cb2 namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: ae6721d5-6c60-4b4c-b21a-b41618d16529 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-fc189 namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: 6735630c-ac48-45fc-b119-d801618c74eb kind: Event lastTimestamp: "2026-09-17T23:43:53Z" message: Scaled up replica set splitter-graph-fc189-8457d9b884 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:53Z" name: splitter-graph-fc189.18d640547ce12edc namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: a5f48658-4ee3-4557-b951-528597c4c8f8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-fc189 namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 4efc026b-ee45-4541-bd9f-83c0585cbd41 kind: Event lastTimestamp: "2026-09-17T23:43:53Z" message: InferenceGraph [splitter-graph-fc189] is Ready metadata: creationTimestamp: "2026-09-17T23:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:43:53Z" name: splitter-graph-fc189.18d64054851c2325 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: dd724ec5-f4c0-4185-b7a2-cffc19907d69 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:23:03.447214Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13295" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-11096-predictor-587bc899cb-dtvwk to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:23:03Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f316d5ec1e4 namespace: kserve-ci-e2e-test resourceVersion: "13299" uid: 74763e38-774e-4980-adad-ef09e38644c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13308" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f318f47c1ab namespace: kserve-ci-e2e-test resourceVersion: "13341" uid: a61ace17-65c1-42db-8f34-792138e9608c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f3190b78717 namespace: kserve-ci-e2e-test resourceVersion: "13343" uid: e3480c73-6328-49f6-babb-5df0d1eccab6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f3193803ba9 namespace: kserve-ci-e2e-test resourceVersion: "13344" uid: 1209ce81-fdd0-4028-a3fd-8a84f772194b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f3193f8e335 namespace: kserve-ci-e2e-test resourceVersion: "13345" uid: f98500f4-febb-4a2a-b144-e45a1d60a260 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f3194041957 namespace: kserve-ci-e2e-test resourceVersion: "13346" uid: 2f0439a4-d150-42e9-8c25-d5e406154340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f319729991e namespace: kserve-ci-e2e-test resourceVersion: "13351" uid: c6b11a44-60d0-4dcc-81dc-15a03b612958 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:23:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:04Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f3197a206d7 namespace: kserve-ci-e2e-test resourceVersion: "13352" uid: a2abda69-b57b-4a35-9280-b1d5270e05f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T23:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:32:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:20Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63f31a1809cee namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 3aed86f7-3daa-4275-9707-a52ba1ffc34f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63fb293a910b9 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 81bdf8db-221b-48c9-8b6b-c39257cb7a7c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63fb293a9be5b namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 5afc1d14-0afa-48c2-b3fd-8de9acdb0eea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11096-predictor-587bc899cb-dtvwk namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: ccbe12c0-40dc-4b43-884f-1680d643c069 kind: Event lastTimestamp: "2026-09-17T23:32:20Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:32:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:20Z" name: success-200-isvc-11096-predictor-587bc899cb-dtvwk.18d63fb315f2d0a6 namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: ef0deda5-ae6e-4074-989e-bbc365dce796 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-11096-predictor-587bc899cb namespace: kserve-ci-e2e-test resourceVersion: "13287" uid: 18701bb9-a004-4b88-aedf-f2d834fdbc48 kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: 'Created pod: success-200-isvc-11096-predictor-587bc899cb-dtvwk' metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:03Z" name: success-200-isvc-11096-predictor-587bc899cb.18d63f316cdaa6bf namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: b67675dd-7a48-459c-9bce-071b34c1cb9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13285" uid: 9fd2485d-5333-4cae-a0dd-d9740cb680f6 kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Scaled up replica set success-200-isvc-11096-predictor-587bc899cb from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:03Z" name: success-200-isvc-11096-predictor.18d63f3168ae3ed7 namespace: kserve-ci-e2e-test resourceVersion: "13290" uid: 11d988d3-6991-43ff-8d35-81b074e136ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:23:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13286" uid: a96afed2-97d5-4184-b0d1-efe151b7bdd9 kind: Event lastTimestamp: "2026-09-17T23:23:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:48Z" name: success-200-isvc-11096-predictor.18d63f34e72d946a namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 2da977be-3ba4-488c-884e-c21feb17a1fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:23:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13286" uid: a96afed2-97d5-4184-b0d1-efe151b7bdd9 kind: Event lastTimestamp: "2026-09-17T23:23:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:48Z" name: success-200-isvc-11096-predictor.18d63f34e79ee0e6 namespace: kserve-ci-e2e-test resourceVersion: "13686" uid: 347bb6f6-0b63-421f-bd27-e27c475a1642 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:23:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13467" uid: a96afed2-97d5-4184-b0d1-efe151b7bdd9 kind: Event lastTimestamp: "2026-09-17T23:24:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:18Z" name: success-200-isvc-11096-predictor.18d63f3866da88d3 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 3683b86f-8d45-4b7a-a92b-df6cb4d45d36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:23:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11096-predictor namespace: kserve-ci-e2e-test resourceVersion: "13467" uid: a96afed2-97d5-4184-b0d1-efe151b7bdd9 kind: Event lastTimestamp: "2026-09-17T23:24:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:18Z" name: success-200-isvc-11096-predictor.18d63f3867b9d4d2 namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: 44a67a6b-2b82-48fb-9c9b-acc9cee2ef9c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11096 namespace: kserve-ci-e2e-test resourceVersion: "13274" uid: f00cc266-5bff-4342-9177-27422e4aa519 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:50Z" name: success-200-isvc-11096.18d63f3168a9db36 namespace: kserve-ci-e2e-test resourceVersion: "13833" uid: db2aacd9-e176-4b35-ad8b-fa59d86ab595 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11096 namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: f00cc266-5bff-4342-9177-27422e4aa519 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: InferenceService [success-200-isvc-11096] is Ready metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:23:50Z" name: success-200-isvc-11096.18d63f3c62b2bb9e namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: 583bd48b-3f86-4bb5-a014-c5259ce05e0f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:23:50.069127Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2026c-predictor-866c975b8d-dr7dv to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:23:50Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3c48409c7d namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: ecd2af1b-0660-4df7-b9bf-dc7dadc0b78a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2026c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:50Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3c4f4601f6 namespace: kserve-ci-e2e-test resourceVersion: "13834" uid: ddd3094d-39c9-4fba-8ed7-1dc6145d9902 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3cc378761b namespace: kserve-ci-e2e-test resourceVersion: "13868" uid: 13bc2a9d-a2b4-4b21-9da7-3634dcc95cd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3cc566cd66 namespace: kserve-ci-e2e-test resourceVersion: "13870" uid: 8203f02e-7af6-43b3-8185-0a113a96f1db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3cc8a5c108 namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: f0aeddea-5712-430c-af04-56f24afa57d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3cc914cb0e namespace: kserve-ci-e2e-test resourceVersion: "13872" uid: 596fd397-a4d9-4136-80f0-b9be91264fc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3cc91ee95b namespace: kserve-ci-e2e-test resourceVersion: "13873" uid: c93b25c9-d936-4dda-b05d-a3e6720e5571 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3ccc45d149 namespace: kserve-ci-e2e-test resourceVersion: "13875" uid: 3fdb1c92-17f9-4d06-af30-c648b38e9bce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:23:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:52Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3cccb1eb98 namespace: kserve-ci-e2e-test resourceVersion: "13876" uid: 68b06d2c-5cd4-4a45-9049-74d695f1a243 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:23:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:24:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:29Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63f3d16ef39f0 namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: 48c5f81a-691c-48c6-a750-7a98caf9e8d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:04Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63fbd62874680 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: bedb42fe-2b68-4ff4-89c9-a01837d1816b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 20639def-b323-4067-ade5-b36de99fa233 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:04Z" name: success-200-isvc-2026c-predictor-866c975b8d-dr7dv.18d63fbd6287ee75 namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 0f9a21b4-dc30-4f10-8f96-c1a51d58c189 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2026c-predictor-866c975b8d namespace: kserve-ci-e2e-test resourceVersion: "13777" uid: 51b82892-90d5-49a7-8b5b-bdc9923cd431 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: 'Created pod: success-200-isvc-2026c-predictor-866c975b8d-dr7dv' metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:50Z" name: success-200-isvc-2026c-predictor-866c975b8d.18d63f3c478b1861 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 119b5137-9ecf-406a-9496-6e2c72020957 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13775" uid: 5a4542b0-8679-43bc-9fd2-e94156994369 kind: Event lastTimestamp: "2026-09-17T23:23:50Z" message: Scaled up replica set success-200-isvc-2026c-predictor-866c975b8d from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:23:50Z" name: success-200-isvc-2026c-predictor.18d63f3c44678de6 namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: d3830cdd-9815-4e48-a9e6-971e0d0bc2b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13776" uid: 8517c614-451d-4a09-90ba-c5242d29979f kind: Event lastTimestamp: "2026-09-17T23:24:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:35Z" name: success-200-isvc-2026c-predictor.18d63f3fc4b7afa2 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: fe04ddb8-a667-4fd8-87fa-1788255dd501 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13776" uid: 8517c614-451d-4a09-90ba-c5242d29979f kind: Event lastTimestamp: "2026-09-17T23:24:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:35Z" name: success-200-isvc-2026c-predictor.18d63f3fc5216626 namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 2665be2c-30d9-4910-8a08-8999dd8d491d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 8517c614-451d-4a09-90ba-c5242d29979f kind: Event lastTimestamp: "2026-09-17T23:25:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:25:20Z" name: success-200-isvc-2026c-predictor.18d63f4a429566d8 namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: c6c7e6f3-0652-4a41-a933-164ed7a9689d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2026c-predictor namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 8517c614-451d-4a09-90ba-c5242d29979f kind: Event lastTimestamp: "2026-09-17T23:25:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:25:20Z" name: success-200-isvc-2026c-predictor.18d63f4a43d8eca7 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: ccb0ed9c-917c-4cf5-90ce-65a532d5d179 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:23:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2026c namespace: kserve-ci-e2e-test resourceVersion: "13760" uid: 194c385b-6b72-4ddb-b0ae-3202c00789eb kind: Event lastTimestamp: "2026-09-17T23:24:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:39Z" name: success-200-isvc-2026c.18d63f3c445525b9 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: bfb798ed-8a20-4a76-9bdb-021aff25c735 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2026c namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 194c385b-6b72-4ddb-b0ae-3202c00789eb kind: Event lastTimestamp: "2026-09-17T23:24:39Z" message: InferenceService [success-200-isvc-2026c] is Ready metadata: creationTimestamp: "2026-09-17T23:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:39Z" name: success-200-isvc-2026c.18d63f47d2f57ada namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 08b8ca5a-a336-401a-9493-c1090397a98e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:32:18.289915Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:32:18Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb29c926988 namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 102a75f4-8b55-4cc4-85f2-81df8cc7ac4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3e634-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2a6b44e3c namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 033e450c-28f9-436b-a979-9854d1e92877 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2e246e5ed namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 2b86fac3-06da-45f6-8685-364fe5c18e60 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2e3d5c5f5 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 3202e648-15f7-41a2-8a95-c1bd667340f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2e6edac9b namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 07eca293-8783-4499-ad2a-cae6c5eb8784 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2e75d8bb0 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: ecdfc2fd-70e5-4aab-97a4-6fbf2d709a6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2e76930dc namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: c4bf09d6-ede9-4c9d-8e1c-0ca952e249bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2ea4478f4 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: 979e0e0c-8d41-4e9b-80bf-74f249f9d16f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:19Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2eab32399 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 41eacd7e-4d8f-4c4e-ab67-a734028669b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:32:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:55Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fb2fa9942b4 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: dbc13cd0-0315-4612-9194-dfb73a54bb1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:28Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fc2f1c7b614 namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: 7017a0d3-b6b9-483b-9f59-763d13cfffa4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:28Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fc2f1c83616 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 77ee3714-cac4-4fcb-bc1a-740f4d7e6ca7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 742c20f3-8c32-4da8-81b2-9052585f616f kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s.18d63fc382e2d86c namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: cd11b3d8-a535-44bf-a2d4-1015b0cf0e4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3e634-predictor-6bd488c9d4 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 939d4520-c9c8-454f-a3b5-e4c155fb6423 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: 'Created pod: success-200-isvc-3e634-predictor-6bd488c9d4-k5j9s' metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:32:18Z" name: success-200-isvc-3e634-predictor-6bd488c9d4.18d63fb29c22829d namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: 6a3dbedc-e195-46aa-8621-68459f2963b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: be80b450-2de5-4a63-b74f-9097e640c6e7 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Scaled up replica set success-200-isvc-3e634-predictor-6bd488c9d4 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:32:18Z" name: success-200-isvc-3e634-predictor.18d63fb29884056a namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 17fd557b-90be-486c-a741-3a8cf0cbcf7f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 1d37d6f9-b5d8-444d-86fd-25fbdce4e057 kind: Event lastTimestamp: "2026-09-17T23:33:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:03Z" name: success-200-isvc-3e634-predictor.18d63fb616e512c8 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 9399fdb7-352a-48c8-bdf4-3f38b6068d0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 1d37d6f9-b5d8-444d-86fd-25fbdce4e057 kind: Event lastTimestamp: "2026-09-17T23:33:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:03Z" name: success-200-isvc-3e634-predictor.18d63fb61750d23a namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 82ecd860-f745-4698-9b09-1317b819bf7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 1d37d6f9-b5d8-444d-86fd-25fbdce4e057 kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:18Z" name: success-200-isvc-3e634-predictor.18d63fb995fc6066 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: e6e40afa-5d3d-46f8-881f-9922a031c4b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:32:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e634-predictor namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 1d37d6f9-b5d8-444d-86fd-25fbdce4e057 kind: Event lastTimestamp: "2026-09-17T23:33:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:18Z" name: success-200-isvc-3e634-predictor.18d63fb996718480 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: d58314f3-2c6e-4180-bc46-f469dcb2c964 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e634 namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: f2747340-f758-42e6-ae4b-7e286abe8bdd kind: Event lastTimestamp: "2026-09-17T23:33:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:06Z" name: success-200-isvc-3e634.18d63fb29884dd49 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: d557d460-ec1c-4f06-8695-95969534ea1d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e634 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: f2747340-f758-42e6-ae4b-7e286abe8bdd kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: InferenceService [success-200-isvc-3e634] is Ready metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-3e634.18d63fbdb702de91 namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: d4987ac3-8ef2-4f14-8fc4-067d29b749cd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:33:04.700545Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:33:04Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd6adc6f55 namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: df9a646a-d21d-4b6b-98b1-29c580d3b17a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd8ca99005 namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: 2806b681-8a0c-4090-8c2f-2210ee9951f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd8e2e1887 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: e6f6c5e8-3b1d-4d39-a51d-3baa85f9e172 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd90d64bfd namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: b8389e18-e3a1-4901-a750-f127fe899c75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd91425a06 namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 2cd9bd6a-065b-4e08-816c-0454ff96a5b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd914ba213 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 5899649a-8c84-43d7-aa7b-bad82eaa07ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd93cbc3aa namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 14cfd5b9-a42a-4433-a5dc-27750b1edb7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:05Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbd9440fc35 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: d4017e16-11cb-4703-adb2-b57100e70aed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:33:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:43Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fbdf9459c16 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 34f48dd9-24d6-451e-905e-662365729719 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:14Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fcdc16678ee namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: 4fefb66a-e829-4d54-b915-bf19799b3e91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 39644fba-18d3-471e-ac91-efe1b9de34df kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:14Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r.18d63fcdc1678167 namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: 0cdeebb7-732c-47da-a0b1-39dd4ad3e6c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4a1f9-predictor-5456b64ccb namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 98c3d103-730f-4d39-803d-e96ebea271df kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: 'Created pod: success-200-isvc-4a1f9-predictor-5456b64ccb-vpv2r' metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:04Z" name: success-200-isvc-4a1f9-predictor-5456b64ccb.18d63fbd6a6c1108 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: b652b1c9-ff2d-4f23-aac1-88eeae4d1366 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: bbba5ac5-676d-41df-9280-d022695ffda9 kind: Event lastTimestamp: "2026-09-17T23:33:04Z" message: Scaled up replica set success-200-isvc-4a1f9-predictor-5456b64ccb from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:04Z" name: success-200-isvc-4a1f9-predictor.18d63fbd681ab445 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 3ebdd88e-63e4-4507-bcb4-6e33dabc4fe0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: d3751e1f-883d-42a7-a67b-bd9b401e8962 kind: Event lastTimestamp: "2026-09-17T23:33:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:49Z" name: success-200-isvc-4a1f9-predictor.18d63fc0e698ab98 namespace: kserve-ci-e2e-test resourceVersion: "17249" uid: 5890d7c3-fcb0-49bd-b018-c59831964cfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: d3751e1f-883d-42a7-a67b-bd9b401e8962 kind: Event lastTimestamp: "2026-09-17T23:33:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:49Z" name: success-200-isvc-4a1f9-predictor.18d63fc0e702b410 namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: 81982922-381f-4c4b-b2d8-c7bb48457074 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: d3751e1f-883d-42a7-a67b-bd9b401e8962 kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:04Z" name: success-200-isvc-4a1f9-predictor.18d63fc46629a300 namespace: kserve-ci-e2e-test resourceVersion: "17365" uid: c300f668-047e-4679-9dab-22622bf49f74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a1f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: d3751e1f-883d-42a7-a67b-bd9b401e8962 kind: Event lastTimestamp: "2026-09-17T23:34:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:04Z" name: success-200-isvc-4a1f9-predictor.18d63fc466916ddf namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: 7c3f999e-4544-4f73-a4f9-dc2fe5717235 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a1f9 namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 1a3f0005-a2ed-4a5b-a8b6-028983ad544d kind: Event lastTimestamp: "2026-09-17T23:33:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:53Z" name: success-200-isvc-4a1f9.18d63fbd681e2800 namespace: kserve-ci-e2e-test resourceVersion: "17283" uid: fdf0d627-d0ff-430a-b551-5a963c1f47a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a1f9 namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 1a3f0005-a2ed-4a5b-a8b6-028983ad544d kind: Event lastTimestamp: "2026-09-17T23:33:53Z" message: InferenceService [success-200-isvc-4a1f9] is Ready metadata: creationTimestamp: "2026-09-17T23:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:33:53Z" name: success-200-isvc-4a1f9.18d63fc8b8148350 namespace: kserve-ci-e2e-test resourceVersion: "17282" uid: d9b4f199-e22c-49c9-bd11-16dfddf727b4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:34:14.995408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-51963-predictor-55b958c4f8-6sthm to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:34:14Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fcdc8c2feae namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 2a7b72d0-4e7e-461c-bb90-514cb8eb9d74 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-51963-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:15Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fcdd532d820 namespace: kserve-ci-e2e-test resourceVersion: "17550" uid: f15677af-0efa-4309-9a1f-b06c653d0ad9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17524" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce0e61eb09 namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: b0eb2dcf-d378-4aab-9872-70aed29c6bbe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce0fb498a2 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: ca9a6399-65ef-49ec-9c90-394bddbd8c8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce124c0bc7 namespace: kserve-ci-e2e-test resourceVersion: "17577" uid: aa213677-6092-4a1e-8030-1133360e47c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce12bcd3ae namespace: kserve-ci-e2e-test resourceVersion: "17578" uid: f341fb6a-6b20-4d56-999a-6eabffd75650 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce12c6d273 namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: 1b96372e-8bb4-4a5f-b93b-fa0551101dcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce15da64c5 namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: 66ac3048-39d2-4db9-9fe4-8055c8aeb89f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:34:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:16Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce16490def namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: e5b76054-1f2d-4474-b011-3bb8a5340aa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T23:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:43:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:32Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d63fce1e70e48e namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: 23580b5d-a251-4413-b680-b3613e774f8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:29Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d6404ef97525c8 namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 4f279b61-69c4-4b7a-a1fc-0ffa17bdf845 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:29Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d6404ef975c8ec namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 631c3dc5-dee0-4e19-b2d8-fab4bc36e5fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-51963-predictor-55b958c4f8-6sthm namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 835d0960-08b8-47c9-bf25-8b4df6b73001 kind: Event lastTimestamp: "2026-09-17T23:43:32Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:32Z" name: success-200-isvc-51963-predictor-55b958c4f8-6sthm.18d6404f93280c09 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 1096b35f-d53e-4043-b61b-e7625df187a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-51963-predictor-55b958c4f8 namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 26c6e034-3f59-4663-bbd6-a4d0fb8781d0 kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: 'Created pod: success-200-isvc-51963-predictor-55b958c4f8-6sthm' metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:14Z" name: success-200-isvc-51963-predictor-55b958c4f8.18d63fcdc84b77b7 namespace: kserve-ci-e2e-test resourceVersion: "17515" uid: 1e94be44-cd1f-4d06-85d9-bbdf483c3f41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: b45fe6f1-ed16-4777-aa1a-08f3f9293f99 kind: Event lastTimestamp: "2026-09-17T23:34:14Z" message: Scaled up replica set success-200-isvc-51963-predictor-55b958c4f8 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:14Z" name: success-200-isvc-51963-predictor.18d63fcdc67f4bbc namespace: kserve-ci-e2e-test resourceVersion: "17509" uid: c59573d6-c135-44b0-8f77-3a11c8f7116b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:34:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 9b967436-7bd6-4f0a-a6f2-caac65d94fb2 kind: Event lastTimestamp: "2026-09-17T23:34:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:44Z" name: success-200-isvc-51963-predictor.18d63fd144ee42ba namespace: kserve-ci-e2e-test resourceVersion: "17811" uid: 97cd8a86-67bc-4809-9a1b-20a92228366b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:34:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 9b967436-7bd6-4f0a-a6f2-caac65d94fb2 kind: Event lastTimestamp: "2026-09-17T23:34:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:44Z" name: success-200-isvc-51963-predictor.18d63fd14554dcef namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: ce0d0168-f421-4678-b33d-bf8fa08d2858 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:34:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 9b967436-7bd6-4f0a-a6f2-caac65d94fb2 kind: Event lastTimestamp: "2026-09-17T23:35:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:30Z" name: success-200-isvc-51963-predictor.18d63fd842c070e4 namespace: kserve-ci-e2e-test resourceVersion: "18093" uid: 1953308c-330e-4164-95fb-a3076eb56477 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:34:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-51963-predictor namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 9b967436-7bd6-4f0a-a6f2-caac65d94fb2 kind: Event lastTimestamp: "2026-09-17T23:35:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:35:30Z" name: success-200-isvc-51963-predictor.18d63fd8432cf0b1 namespace: kserve-ci-e2e-test resourceVersion: "18094" uid: 72241e41-179a-4990-8acc-c4ee6115d5d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:34:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-51963 namespace: kserve-ci-e2e-test resourceVersion: "17496" uid: ca1a7dd4-33af-49cc-95b0-d34c7733877e kind: Event lastTimestamp: "2026-09-17T23:35:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:35:02Z" name: success-200-isvc-51963.18d63fcdc67dc941 namespace: kserve-ci-e2e-test resourceVersion: "17913" uid: a051f370-ee24-4a7d-8dfd-33938374bb98 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:35:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-51963 namespace: kserve-ci-e2e-test resourceVersion: "17911" uid: ca1a7dd4-33af-49cc-95b0-d34c7733877e kind: Event lastTimestamp: "2026-09-17T23:35:02Z" message: InferenceService [success-200-isvc-51963] is Ready metadata: creationTimestamp: "2026-09-17T23:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:35:02Z" name: success-200-isvc-51963.18d63fd8d9c355a3 namespace: kserve-ci-e2e-test resourceVersion: "17912" uid: f66303f4-a9fe-4192-9b97-f4bdc3b7e029 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:43:30.004336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:43:30Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f01df2553 namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: df630cc6-7b35-4e06-9f4e-77e1973e3c3b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-57566-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:30Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f0fbe2901 namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 1c3282d0-2ed8-4098-8197-ada12508183a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f4818dfdc namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: c8e6a219-dc06-4e46-b4a8-609a67baeef5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f497c795c namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 63d1fc22-9c1f-4847-b084-1e820761f7ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f4d38df26 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: b7210d4e-37ab-449c-9017-e3f149c56dc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f4dd8fb64 namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 3bd64d2e-b86e-4f65-a38e-457f658e41d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f4de89cb1 namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 07894429-980a-49ec-992c-1f6cc7bdc250 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f529d4428 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: da256149-f590-474a-937a-10b5746cef18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404f5325f61f namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 895296b4-a522-4e29-9a0c-f10d54ffb0d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T23:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-17T23:44:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:18Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d6404faf1a3e3c namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 244acd7b-8a18-4978-a476-e87fa6560b1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T00:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-18T00:00:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T00:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T00:00:59Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d641435ba762ae namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: 5b5dfbd2-fc20-4efa-adc2-9fd4e9f18871 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T00:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 1c151445-b6ed-41c4-9e32-362ee62596ed kind: Event lastTimestamp: "2026-09-18T00:00:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T00:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T00:00:59Z" name: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7.18d641435ba7f7b9 namespace: kserve-ci-e2e-test resourceVersion: "25876" uid: 2ac99957-5eec-4d05-859a-461782a26bf8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-57566-predictor-6949f9f8f6 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 0eb1c99c-2a40-4976-9ccb-e0b35db851c3 kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: 'Created pod: success-200-isvc-57566-predictor-6949f9f8f6-9bgg7' metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:29Z" name: success-200-isvc-57566-predictor-6949f9f8f6.18d6404f015f08de namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 690fedb5-a83e-4c7a-8c80-5bce011b2919 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: e9d20a56-00d3-4b9d-b721-f00a3d79754e kind: Event lastTimestamp: "2026-09-17T23:43:29Z" message: Scaled up replica set success-200-isvc-57566-predictor-6949f9f8f6 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:29Z" name: success-200-isvc-57566-predictor.18d6404eff8f2b14 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 51a314cf-2ffc-4403-806f-76f484da0995 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:43:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: e60b4ad6-605b-42d0-b900-d616d3af567a kind: Event lastTimestamp: "2026-09-17T23:43:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:59Z" name: success-200-isvc-57566-predictor.18d640527e02937a namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: dc5f4189-7ff8-48d3-9676-47d7bbfb626e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:43:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: e60b4ad6-605b-42d0-b900-d616d3af567a kind: Event lastTimestamp: "2026-09-17T23:43:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:59Z" name: success-200-isvc-57566-predictor.18d640527e784f13 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 77e47be9-c9bd-4c31-8c22-04564063b224 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:44:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: e60b4ad6-605b-42d0-b900-d616d3af567a kind: Event lastTimestamp: "2026-09-17T23:45:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:00Z" name: success-200-isvc-57566-predictor.18d640597c9c99aa namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: be30b10a-bb8f-4507-af64-44e64bb6b35f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:44:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57566-predictor namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: e60b4ad6-605b-42d0-b900-d616d3af567a kind: Event lastTimestamp: "2026-09-17T23:45:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:00Z" name: success-200-isvc-57566-predictor.18d640597d0c16b6 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 6dc23fa6-0c6f-4ee3-a87a-7e58a3a4bead reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:43:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57566 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 6fa4cf85-a51a-49ff-8961-a0924ecc119d kind: Event lastTimestamp: "2026-09-17T23:44:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:29Z" name: success-200-isvc-57566.18d6404eff8d4f4e namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: 40b3eae5-cddf-469a-944c-c42e3f8957ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57566 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 6fa4cf85-a51a-49ff-8961-a0924ecc119d kind: Event lastTimestamp: "2026-09-17T23:44:29Z" message: InferenceService [success-200-isvc-57566] is Ready metadata: creationTimestamp: "2026-09-17T23:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:29Z" name: success-200-isvc-57566.18d6405cc0061900 namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: dcc5c3fd-d2ab-4c30-9df6-91155e2e98fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:21:29.203137Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12431" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:21:29Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1b7bfc4515 namespace: kserve-ci-e2e-test resourceVersion: "12435" uid: 61308fe9-d0f0-4e3c-af5c-1adc7d61bd03 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b6ba0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:29Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1b83077d5b namespace: kserve-ci-e2e-test resourceVersion: "12459" uid: 1e4661bc-91ff-4906-9b18-c11e40c0e6a3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12439" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:21:30Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1bc2100dbf namespace: kserve-ci-e2e-test resourceVersion: "12554" uid: 4e018510-601c-4b61-af98-de6f7d3cb9cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:30Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" metadata: creationTimestamp: "2026-09-17T23:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:30Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1bc3961598 namespace: kserve-ci-e2e-test resourceVersion: "12557" uid: c8c5eecc-3e40-4ea6-b30a-e796e2ac4fc5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:42Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" in 11.936s (11.936s including waiting). Image size: 1356848233 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:42Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1e8b0e8b50 namespace: kserve-ci-e2e-test resourceVersion: "12651" uid: 864eecb2-d1c4-4bf5-8d02-ebf8a4cef9da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:42Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1e8e39ec8b namespace: kserve-ci-e2e-test resourceVersion: "12652" uid: ece43899-1c72-4ef6-883e-26ea5ed3cd83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:42Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1e8ebaa9f0 namespace: kserve-ci-e2e-test resourceVersion: "12653" uid: 452891b6-5008-4f5b-bcce-9e6e07ab2012 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T23:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:42Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1e8ecb06bb namespace: kserve-ci-e2e-test resourceVersion: "12654" uid: 531b5900-4d46-4fad-a4db-24645d97720d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.338s (2.338s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:44Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1f1a3407e3 namespace: kserve-ci-e2e-test resourceVersion: "12677" uid: 4314cdaa-37be-4aa0-9794-6dddba2e597e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:44Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1f1d5c57d5 namespace: kserve-ci-e2e-test resourceVersion: "12678" uid: b67a3a68-d081-4bb6-95c5-e9716a41fbe5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:21:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:21:44Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1f1de5ba0a namespace: kserve-ci-e2e-test resourceVersion: "12679" uid: 61aff681-b057-4976-be89-276d6a90e59a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:22:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:20Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f1f26ebd004 namespace: kserve-ci-e2e-test resourceVersion: "12918" uid: 2ebfdc59-ef95-441e-bfde-975cbbd4bed8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:03Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f31626b9648 namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: 9f56e33c-5b04-4272-9c55-557a56f3bcbf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:03Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f31626bddb5 namespace: kserve-ci-e2e-test resourceVersion: "13272" uid: 7858ec58-df49-4526-aed9-f4fa0996aea4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t namespace: kserve-ci-e2e-test resourceVersion: "12433" uid: 76aa4e94-21d0-40bb-b318-115624a91754 kind: Event lastTimestamp: "2026-09-17T23:23:05Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:05Z" name: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t.18d63f32031bc287 namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: bb66411d-a121-4971-9b2b-396127409d13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b6ba0-predictor-76bc585db8 namespace: kserve-ci-e2e-test resourceVersion: "12422" uid: 51e086f2-e95d-4440-a4cc-c1c3a554a3fe kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: 'Created pod: success-200-isvc-b6ba0-predictor-76bc585db8-ldc2t' metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: success-200-isvc-b6ba0-predictor-76bc585db8.18d63f1b7b9027d7 namespace: kserve-ci-e2e-test resourceVersion: "12432" uid: 09600038-7b1c-47cb-bb4a-eb9d30e0471d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12420" uid: 3b31f7d3-2540-4f50-a88a-84c68b62ad49 kind: Event lastTimestamp: "2026-09-17T23:21:29Z" message: Scaled up replica set success-200-isvc-b6ba0-predictor-76bc585db8 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:29Z" name: success-200-isvc-b6ba0-predictor.18d63f1b78638627 namespace: kserve-ci-e2e-test resourceVersion: "12423" uid: 75309684-01f7-4d89-b832-ca097743f7ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12421" uid: fa1539b4-2fa0-49e0-9a01-c1e8f603e022 kind: Event lastTimestamp: "2026-09-17T23:21:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:59Z" name: success-200-isvc-b6ba0-predictor.18d63f1ef6fc9bce namespace: kserve-ci-e2e-test resourceVersion: "12789" uid: d1c3d846-2baf-417d-bc5c-cb443dc7c439 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12421" uid: fa1539b4-2fa0-49e0-9a01-c1e8f603e022 kind: Event lastTimestamp: "2026-09-17T23:21:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:21:59Z" name: success-200-isvc-b6ba0-predictor.18d63f1ef777f259 namespace: kserve-ci-e2e-test resourceVersion: "12790" uid: 9fff4a99-e5ab-480d-99b1-54a3b11cd3dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:22:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: fa1539b4-2fa0-49e0-9a01-c1e8f603e022 kind: Event lastTimestamp: "2026-09-17T23:22:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:59Z" name: success-200-isvc-b6ba0-predictor.18d63f25f4f8aa2e namespace: kserve-ci-e2e-test resourceVersion: "13193" uid: 8b74a0da-b7a1-4cbd-9421-0eecb3bcc55c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T23:22:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6ba0-predictor namespace: kserve-ci-e2e-test resourceVersion: "12666" uid: fa1539b4-2fa0-49e0-9a01-c1e8f603e022 kind: Event lastTimestamp: "2026-09-17T23:22:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:59Z" name: success-200-isvc-b6ba0-predictor.18d63f25f5477a18 namespace: kserve-ci-e2e-test resourceVersion: "13194" uid: 6cd86177-7044-4d73-82c8-9e53adc4359d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T23:21:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6ba0 namespace: kserve-ci-e2e-test resourceVersion: "12414" uid: d4bc0487-09d9-4e30-88f7-d75e5b0fc153 kind: Event lastTimestamp: "2026-09-17T23:22:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:22:31Z" name: success-200-isvc-b6ba0.18d63f1b786be270 namespace: kserve-ci-e2e-test resourceVersion: "12998" uid: 136ca098-5035-4342-88c7-6e6e00190fbc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6ba0 namespace: kserve-ci-e2e-test resourceVersion: "12996" uid: d4bc0487-09d9-4e30-88f7-d75e5b0fc153 kind: Event lastTimestamp: "2026-09-17T23:22:31Z" message: InferenceService [success-200-isvc-b6ba0] is Ready metadata: creationTimestamp: "2026-09-17T23:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:22:31Z" name: success-200-isvc-b6ba0.18d63f29e24a4772 namespace: kserve-ci-e2e-test resourceVersion: "12997" uid: 2c014e2d-08c4-4587-808b-fede16f8c1d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:33:28.565693Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:33:28Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc2f955b6b2 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: 39e03b71-2379-4451-af01-859521073a18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d6263-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:29Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc3069e67ed namespace: kserve-ci-e2e-test resourceVersion: "17090" uid: 90431602-ea8a-4694-84ab-7e8ace03e8d9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc374a957bb namespace: kserve-ci-e2e-test resourceVersion: "17108" uid: 2502152f-2b63-4f6a-8908-c1136a247870 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc375f30848 namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 4c7e62a1-d5cf-49bf-a683-db643dcf51cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc378aa8afe namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 38c4eb2e-d4ba-415b-879b-e3fb4b5815ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc3791b5422 namespace: kserve-ci-e2e-test resourceVersion: "17113" uid: b0b06bed-fb26-4394-9cfb-44c25ed2ad92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc379261433 namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: 4f10c715-3901-4084-93a7-e144d65504f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc37d323a3c namespace: kserve-ci-e2e-test resourceVersion: "17115" uid: 96a4c332-f3da-452e-9f9c-fd432d94086c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:33:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:33:30Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc37db57996 namespace: kserve-ci-e2e-test resourceVersion: "17116" uid: 74271bac-6413-4da3-ac6f-f14adae1f880 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:34:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:34:08Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d63fc3d2b9e949 namespace: kserve-ci-e2e-test resourceVersion: "17403" uid: caf5a16e-de7f-4f8d-9231-ae763ecf8f63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:43Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d64044306ba30f namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: a319aff8-ee25-416f-9b4d-f4a575060b68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 31174a96-f041-4e06-878b-0e4487e22ca3 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:43Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj.18d64044306c32ad namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 8ae9bd9f-f353-4882-8862-65e33806376d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d6263-predictor-5bd97bfcc6 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: c44f9499-e8d4-4587-89cf-2c7006c1c2d3 kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: 'Created pod: success-200-isvc-d6263-predictor-5bd97bfcc6-l5bmj' metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:28Z" name: success-200-isvc-d6263-predictor-5bd97bfcc6.18d63fc2f8bb23ca namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: bcd682e4-ca79-48f4-94e1-6ebca814fd41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 6d3c84f6-42ed-4681-9ecd-90b38f37c721 kind: Event lastTimestamp: "2026-09-17T23:33:28Z" message: Scaled up replica set success-200-isvc-d6263-predictor-5bd97bfcc6 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:28Z" name: success-200-isvc-d6263-predictor.18d63fc2f7032c13 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: e667ceb8-1ee0-474a-98d8-6495dd92b5dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: 9508108f-4b4b-4b38-a614-bd22e6c09a14 kind: Event lastTimestamp: "2026-09-17T23:33:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:58Z" name: success-200-isvc-d6263-predictor.18d63fc67572c501 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 65cc6fe6-7879-487f-8dff-d242e7b01a80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:33:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: 9508108f-4b4b-4b38-a614-bd22e6c09a14 kind: Event lastTimestamp: "2026-09-17T23:33:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:33:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:33:58Z" name: success-200-isvc-d6263-predictor.18d63fc675d580c0 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: e527d8df-d29b-4a65-9042-228dde5dcc8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:34:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: 9508108f-4b4b-4b38-a614-bd22e6c09a14 kind: Event lastTimestamp: "2026-09-17T23:34:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:43Z" name: success-200-isvc-d6263-predictor.18d63fcd7344f289 namespace: kserve-ci-e2e-test resourceVersion: "17797" uid: ac6fc035-9ae6-4f70-8bb7-314592e9d238 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:34:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d6263-predictor namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: 9508108f-4b4b-4b38-a614-bd22e6c09a14 kind: Event lastTimestamp: "2026-09-17T23:34:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:34:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:34:43Z" name: success-200-isvc-d6263-predictor.18d63fcd739954f2 namespace: kserve-ci-e2e-test resourceVersion: "17798" uid: 8b4c10ac-6e75-49bd-916f-25440b7ccf64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:33:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 57568e87-4166-4620-96e6-aa2814a88f98 kind: Event lastTimestamp: "2026-09-17T23:34:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:34:18Z" name: success-200-isvc-d6263.18d63fc2f6fdc223 namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: f2c0249e-c0f6-4a4b-864c-5fada5b521ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:34:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d6263 namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: 57568e87-4166-4620-96e6-aa2814a88f98 kind: Event lastTimestamp: "2026-09-17T23:34:18Z" message: InferenceService [success-200-isvc-d6263] is Ready metadata: creationTimestamp: "2026-09-17T23:34:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:34:18Z" name: success-200-isvc-d6263.18d63fce8e77f90b namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: 729fda88-0935-4f43-9f31-2f0b365ffe11 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:44:03.908437Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-daa97-predictor-845479f546-9wxxx to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:44:03Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d64056e6b697a0 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 9ffa3bd7-6782-495c-9292-152c80105cb4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-daa97-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:04Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d64056f2f54bfb namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 85a8784d-447a-48d3-a569-a35c90203092 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640572cfa2446 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 86eba3a8-53aa-4510-b24c-07ed6c86760f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640572f032cb5 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: c20b1838-f6da-4d04-8cbe-ab5b0765fb9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640573287d9b4 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 33768ac9-2a04-4d71-830c-74aa80423e37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d64057330d60e2 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 489971ba-8200-4e88-abc5-194616ee0419 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640573319bdec namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: c185593b-a8b7-4c24-8cc8-e3bbc374bc23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d6405736e39021 namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 284efd18-8267-4045-bb26-fdf6ef3f068f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:44:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:05Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d64057376968b4 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 747e9842-5122-4a4f-bf36-a95d5c0f474a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-17T23:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:53:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:32Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d64057686203a9 namespace: kserve-ci-e2e-test resourceVersion: "23882" uid: 0324e6d8-0bcc-4f76-94eb-bda356d965fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:53:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:53:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:28Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640da67e71cde namespace: kserve-ci-e2e-test resourceVersion: "23852" uid: d801886b-f4d6-4ffe-8473-2812b5f430b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:53:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:53:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:53:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:28Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640da67e7af91 namespace: kserve-ci-e2e-test resourceVersion: "23856" uid: 94121925-6897-4cee-a42a-32f2a5a9d6a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daa97-predictor-845479f546-9wxxx namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: eb2de6d8-6144-4506-aac2-678e98ea3f36 kind: Event lastTimestamp: "2026-09-17T23:53:32Z" message: 'Readiness probe failed: Get "https://10.132.0.45:8643/healthz": dial tcp 10.132.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:53:32Z" name: success-200-isvc-daa97-predictor-845479f546-9wxxx.18d640db30e3cc25 namespace: kserve-ci-e2e-test resourceVersion: "23881" uid: 78df1226-e868-4997-8255-01be831331fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-daa97-predictor-845479f546 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: fcc5859f-a7ab-450f-9124-f85e24b40bc4 kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: 'Created pod: success-200-isvc-daa97-predictor-845479f546-9wxxx' metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:03Z" name: success-200-isvc-daa97-predictor-845479f546.18d64056e600bed6 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 44c6338f-94c7-429c-8908-68d54535e4d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 14f7a11b-e29a-47b5-af25-2d1ae0673f04 kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: Scaled up replica set success-200-isvc-daa97-predictor-845479f546 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:03Z" name: success-200-isvc-daa97-predictor.18d64056e3bb14d8 namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: ab194bd7-8ac3-476f-a771-e64623aef015 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 7dcd17fe-5089-4cb2-b763-def1f383a127 kind: Event lastTimestamp: "2026-09-17T23:44:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:48Z" name: success-200-isvc-daa97-predictor.18d6405a624c1661 namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 4658855b-ab16-40cd-9ac6-c330e238f359 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 7dcd17fe-5089-4cb2-b763-def1f383a127 kind: Event lastTimestamp: "2026-09-17T23:44:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:48Z" name: success-200-isvc-daa97-predictor.18d6405a62b3e9f8 namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 1c6fe39e-1f62-4c2f-98f2-f3e14c0f8ba9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:44:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 7dcd17fe-5089-4cb2-b763-def1f383a127 kind: Event lastTimestamp: "2026-09-17T23:45:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:18Z" name: success-200-isvc-daa97-predictor.18d6405de20474f0 namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: 0102e4ab-9aef-4587-91a5-92473071a071 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:44:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daa97-predictor namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 7dcd17fe-5089-4cb2-b763-def1f383a127 kind: Event lastTimestamp: "2026-09-17T23:45:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:18Z" name: success-200-isvc-daa97-predictor.18d6405de2bf894f namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: dc39fa3e-e668-44ec-a327-274078885f28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: b654e8ac-4831-45be-89bb-7c156f862905 kind: Event lastTimestamp: "2026-09-17T23:45:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:45:02Z" name: success-200-isvc-daa97.18d64056e3c68a3c namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: bda0f763-1feb-4488-b39f-50844ccc024c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-daa97 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: b654e8ac-4831-45be-89bb-7c156f862905 kind: Event lastTimestamp: "2026-09-17T23:45:02Z" message: InferenceService [success-200-isvc-daa97] is Ready metadata: creationTimestamp: "2026-09-17T23:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:45:02Z" name: success-200-isvc-daa97.18d64064787ca71d namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: edf5d920-b38a-46a0-a846-431fe6e032f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:42:43.600753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:42:43Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d64044340099dd namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 3860222f-c174-49dd-ba6e-2bd912763f52 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d64044567dd9a7 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 8334a81f-d7d8-47f6-9587-8a33d5749cef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1976" already present on machine metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d6404457ffd045 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 315b93fd-4442-4a9e-8746-9564acc56e10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d640445b602c64 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 7809d498-4907-453c-a9b8-ed1e38254271 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d640445be96e18 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 7ca4379d-7179-4888-9dca-defe5e3d4669 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d640445bf5561e namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 91c38bb2-89fb-4f9b-a054-41b824638fd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d640445f8faeea namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: e9ccb887-de11-4f42-ab2a-1604498a3a21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:42:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:42:44Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d640446016a070 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 3983242d-6d9b-42f9-a4a2-73a353b226f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T23:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:43:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:43:31Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d64044afee729c namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: b3a26551-b894-47c6-89c8-245ddb86017c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:03Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d64056dddfe911 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 41e3467c-7fd8-402d-b5a1-13249b6f88ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:44:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T23:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:03Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d64056dde0c485 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: b99a9580-b000-44cc-a21f-c36c18954e29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: f4f4541d-671a-4d17-87dc-a4a7471d5268 kind: Event lastTimestamp: "2026-09-17T23:44:06Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T23:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:06Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn.18d640578c23c385 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 0ef87657-60bf-4f77-bc6a-9c0061a9ac9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fc189-predictor-8b7b5ddbd namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 04f5cfdd-315e-40db-8cf3-3ee44fd8099f kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: 'Created pod: success-200-isvc-fc189-predictor-8b7b5ddbd-lscbn' metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:42:43Z" name: success-200-isvc-fc189-predictor-8b7b5ddbd.18d64044330dfa47 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: c27ebe41-81a3-487a-9391-db7081f9f9ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 781ea77d-35d0-42cf-8e8b-7fbb5fb6d141 kind: Event lastTimestamp: "2026-09-17T23:42:43Z" message: Scaled up replica set success-200-isvc-fc189-predictor-8b7b5ddbd from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:42:43Z" name: success-200-isvc-fc189-predictor.18d640442da80bf7 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 56d45c13-5970-45af-bcc6-d651f832ffea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:42:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 6b53eaed-a825-4fa4-a838-06345be2086f kind: Event lastTimestamp: "2026-09-17T23:43:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:13Z" name: success-200-isvc-fc189-predictor.18d64047ac385b54 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: b994f6c2-5f6f-4aa4-b164-01d828352098 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:42:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 6b53eaed-a825-4fa4-a838-06345be2086f kind: Event lastTimestamp: "2026-09-17T23:43:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:42:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:13Z" name: success-200-isvc-fc189-predictor.18d64047aca1ba2d namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: aad978bc-dd35-40cd-b9d9-574caf9d37a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:43:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 6b53eaed-a825-4fa4-a838-06345be2086f kind: Event lastTimestamp: "2026-09-17T23:43:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:58Z" name: success-200-isvc-fc189-predictor.18d6404eaa91cb8b namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 4b0725da-9487-4e35-a04a-16d8d7ffa79f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:43:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc189-predictor namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 6b53eaed-a825-4fa4-a838-06345be2086f kind: Event lastTimestamp: "2026-09-17T23:43:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T23:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:43:58Z" name: success-200-isvc-fc189-predictor.18d6404eab054478 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 925358f5-60b1-4f19-915d-8ded953f7102 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:42:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc189 namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: 82ba404f-7fc8-44c1-9cf6-13a28ab9c14c kind: Event lastTimestamp: "2026-09-17T23:43:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T23:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:43:41Z" name: success-200-isvc-fc189.18d640442da52ec4 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 7b3bebdb-fd86-4b1d-a23f-487c573136c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:43:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc189 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 82ba404f-7fc8-44c1-9cf6-13a28ab9c14c kind: Event lastTimestamp: "2026-09-17T23:43:41Z" message: InferenceService [success-200-isvc-fc189] is Ready metadata: creationTimestamp: "2026-09-17T23:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:43:41Z" name: success-200-isvc-fc189.18d64051bf807fb6 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: fe39ae1d-cbac-4eaf-8af3-34e8ab592405 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:24:03.387569Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13953" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-11096-d974cf6bd-77wlw to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63f3f6217fd16 namespace: kserve-ci-e2e-test resourceVersion: "13958" uid: 439a7a24-7c91-44bf-a9d0-12a7ec66135c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:03Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:24:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-11096-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63f3f6d832d85 namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: 79fcf3d4-9cb9-4fdb-a75a-f85bd8dd1807 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:24:04Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:24:04Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63f3fa7c5740f namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: 65c02dc1-0b1e-4ff3-9fd8-171cf73a8503 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-11096} kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:24:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:04Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63f3fa90dbdfb namespace: kserve-ci-e2e-test resourceVersion: "13987" uid: 8d9b3c32-4133-471e-87a0-1dc22fe86b27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-11096} kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:24:04Z" message: 'Created container: switch-graph-11096' metadata: creationTimestamp: "2026-09-17T23:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:04Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63f3facbe79d0 namespace: kserve-ci-e2e-test resourceVersion: "13988" uid: 7fcb93b3-46eb-439e-b5fc-ef62109932e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-11096} kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:24:04Z" message: Started container switch-graph-11096 metadata: creationTimestamp: "2026-09-17T23:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:24:04Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63f3fad2cd52f namespace: kserve-ci-e2e-test resourceVersion: "13989" uid: 1555b90c-97ed-4543-9dc3-1c3e79baddf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-11096} kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:32:18Z" message: Stopping container switch-graph-11096 metadata: creationTimestamp: "2026-09-17T23:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:18Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63fb28bbeb7f6 namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: f048b43b-bef5-4a42-9be3-d534e5144bc2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:32:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-11096} kind: Pod name: switch-graph-11096-d974cf6bd-77wlw namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 9ae51a90-c525-49e0-a1f8-d2c95f7194b9 kind: Event lastTimestamp: "2026-09-17T23:32:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:32:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:32:46Z" name: switch-graph-11096-d974cf6bd-77wlw.18d63fb361fc7187 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 7662f975-8b90-4af9-9cff-f0f826a7b5f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-11096-d974cf6bd namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: f7e0fb06-66b4-4d48-b4c2-d9c77c147b54 kind: Event lastTimestamp: "2026-09-17T23:24:03Z" message: 'Created pod: switch-graph-11096-d974cf6bd-77wlw' metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096-d974cf6bd.18d63f3f616ae309 namespace: kserve-ci-e2e-test resourceVersion: "13955" uid: 7f839ef0-7202-4c27-9c11-0f0f4b14ac53 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-11096 namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: 9e90a2ca-25cc-451c-94b8-5f4d55daa748 kind: Event lastTimestamp: "2026-09-17T23:24:03Z" message: Scaled up replica set switch-graph-11096-d974cf6bd from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096.18d63f3f6029d5c7 namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: acdb3bb5-84f1-4ed9-bc35-99122212ba25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-11096 namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 1e94d947-2a9a-4d56-8991-d380cb83ce79 kind: Event lastTimestamp: "2026-09-17T23:24:03Z" message: InferenceGraph [switch-graph-11096] is Ready metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096.18d63f3f6a19ac4c namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: a0307ba6-a51c-4508-b842-4df8c5425118 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-11096 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 1e94d947-2a9a-4d56-8991-d380cb83ce79 kind: Event lastTimestamp: "2026-09-17T23:24:03Z" message: 'Failed to update status for InferenceGraph "switch-graph-11096": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-11096": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096.18d63f3f6e49d932 namespace: kserve-ci-e2e-test resourceVersion: "13974" uid: 530fd193-86d4-4dde-8e98-1de7379283f6 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-11096 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 1e94d947-2a9a-4d56-8991-d380cb83ce79 kind: Event lastTimestamp: "2026-09-17T23:24:03Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-11096": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T23:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:24:03Z" name: switch-graph-11096.18d63f3f6e4a1e12 namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: 08f26363-9851-4f0c-857c-5efdaf8836b2 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-11096 namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 503a01c2-0dc3-465c-ac30-efc9da5384dc kind: Event lastTimestamp: "2026-09-17T23:24:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:48Z" name: switch-graph-11096.18d63f42dec7631c namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 7ada1569-26ee-4b6a-95e8-5283763f36d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T23:24:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-11096 namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 503a01c2-0dc3-465c-ac30-efc9da5384dc kind: Event lastTimestamp: "2026-09-17T23:24:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:24:48Z" name: switch-graph-11096.18d63f42df3056c2 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: 47f49318-70cb-4633-b7cd-03bc8fb92f5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:44:40.016612Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-57566-869f5ff548-h6h58 to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548-h6h58.18d6405f4eedafc2 namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: f08a52f6-dfa5-44cd-ae8a-50b9840b0d6b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-57566-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548-h6h58.18d6405f5590b2f8 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: e21c185b-732b-46ef-95af-816059b5fc3f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548-h6h58.18d6405f8343816e namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 2488ba5c-059c-452e-a635-bd36c22d6521 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-57566} kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" already present on machine metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548-h6h58.18d6405f84b202d3 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 1a2d8539-bfb9-4038-8607-c2b14310d277 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-57566} kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: 'Created container: switch-graph-57566' metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548-h6h58.18d6405f889c3c7d namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: fbb7801b-3f63-4644-b40c-662fb3c4b2da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-57566} kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: Started container switch-graph-57566 metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548-h6h58.18d6405f891b9f24 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 18efc7e9-7587-4d16-b659-97d7caa3333f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T00:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-57566} kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-18T00:00:59Z" message: Stopping container switch-graph-57566 metadata: creationTimestamp: "2026-09-18T00:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T00:00:59Z" name: switch-graph-57566-869f5ff548-h6h58.18d64143548fd26a namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: a1d0ebfd-89d1-4a65-8fff-441d6bc0c7dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T00:01:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-57566} kind: Pod name: switch-graph-57566-869f5ff548-h6h58 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 8d4492c4-73ec-4ea0-b22f-76077549f5c3 kind: Event lastTimestamp: "2026-09-18T00:01:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T00:01:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T00:01:27Z" name: switch-graph-57566-869f5ff548-h6h58.18d64144014f10c0 namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 27d7ca77-5fab-4d14-8f31-9b5b1987a095 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-57566-869f5ff548 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 4a5cb908-f4ec-4187-971a-2397a7450a29 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: 'Created pod: switch-graph-57566-869f5ff548-h6h58' metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566-869f5ff548.18d6405f4e71d5a6 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 680a2846-ee91-4dd5-8e19-f442a7c63968 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-57566 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 7f890b5d-fcb4-48e5-8e92-dabd8c2710d8 kind: Event lastTimestamp: "2026-09-17T23:44:39Z" message: Scaled up replica set switch-graph-57566-869f5ff548 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:44:39Z" name: switch-graph-57566.18d6405f4d729caf namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 1f9b62d6-14ea-4a83-800b-38e154d4176b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:44:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-57566 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: f3987b2c-161d-46ba-acbc-4a91795ddd14 kind: Event lastTimestamp: "2026-09-17T23:44:40Z" message: InferenceGraph [switch-graph-57566] is Ready metadata: creationTimestamp: "2026-09-17T23:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:44:40Z" name: switch-graph-57566.18d6405f532bc004 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 97eef189-0439-4a88-832a-9795291fcf84 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-57566 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 08a44a1b-9d38-49b6-b568-53a1289dc86c kind: Event lastTimestamp: "2026-09-17T23:45:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:10Z" name: switch-graph-57566.18d64062cc01fba4 namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 77f2532f-a008-4349-ad55-b4bb555db908 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-57566 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 08a44a1b-9d38-49b6-b568-53a1289dc86c kind: Event lastTimestamp: "2026-09-17T23:45:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T23:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:45:10Z" name: switch-graph-57566.18d64062cc76742b namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: e445a8b4-57f0-4490-a623-7e94772b8d6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T23:22:48.940677Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13111" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b6ba0-69bcc565d7-ct95z to ip-10-0-142-96.ec2.internal metadata: creationTimestamp: "2026-09-17T23:22:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T23:22:48Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2e0cb68b62 namespace: kserve-ci-e2e-test resourceVersion: "13116" uid: 045caf6c-2761-4eb6-aa3b-7a2f887812d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-648d989bfb-n9q9f source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T23:22:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:22:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-b6ba0-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T23:22:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:49Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2e167fb63a namespace: kserve-ci-e2e-test resourceVersion: "13133" uid: 805f2333-a9d4-4162-84ca-263aca72f314 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13120" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:22:51Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T23:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T23:22:51Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2e88487c3c namespace: kserve-ci-e2e-test resourceVersion: "13142" uid: f2a70fce-2897-4083-b375-3a5343d5f6de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b6ba0} kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:22:51Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" metadata: creationTimestamp: "2026-09-17T23:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:51Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2e89a1408a namespace: kserve-ci-e2e-test resourceVersion: "13144" uid: d90abf8e-e52b-4af2-8357-00cca0116494 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b6ba0} kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:22:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:72663b376a728834b8b7a0627c1f77261896d78389c8bb03de412e525cf47359" in 2.294s (2.294s including waiting). Image size: 211751048 bytes.' metadata: creationTimestamp: "2026-09-17T23:22:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:53Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2f126555ca namespace: kserve-ci-e2e-test resourceVersion: "13158" uid: 6530cdc2-ff36-4c48-869a-6a7bd730ec2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b6ba0} kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:22:53Z" message: 'Created container: switch-graph-b6ba0' metadata: creationTimestamp: "2026-09-17T23:22:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:53Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2f165f25e8 namespace: kserve-ci-e2e-test resourceVersion: "13160" uid: 0a9425bf-ee23-40eb-9c4b-fac8958dab7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b6ba0} kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:22:53Z" message: Started container switch-graph-b6ba0 metadata: creationTimestamp: "2026-09-17T23:22:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:22:53Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f2f16ddf571 namespace: kserve-ci-e2e-test resourceVersion: "13161" uid: 6917aebe-d8a9-4509-92e4-73da49ef9b38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b6ba0} kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:23:03Z" message: Stopping container switch-graph-b6ba0 metadata: creationTimestamp: "2026-09-17T23:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:03Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f3157a065f5 namespace: kserve-ci-e2e-test resourceVersion: "13236" uid: 695f8091-0bcc-437d-b74d-61120f4ea8c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T23:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b6ba0} kind: Pod name: switch-graph-b6ba0-69bcc565d7-ct95z namespace: kserve-ci-e2e-test resourceVersion: "13114" uid: bf9844c5-fd16-4bee-8a28-94212e619177 kind: Event lastTimestamp: "2026-09-17T23:23:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T23:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T23:23:30Z" name: switch-graph-b6ba0-69bcc565d7-ct95z.18d63f31da6ab43d namespace: kserve-ci-e2e-test resourceVersion: "13542" uid: 41f06945-b15b-4948-86c7-e9a3bf885ae4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-96.ec2.internal source: component: kubelet host: ip-10-0-142-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b6ba0-69bcc565d7 namespace: kserve-ci-e2e-test resourceVersion: "13108" uid: 5cbbe497-b0d0-4b3b-9744-281e48cb86e5 kind: Event lastTimestamp: "2026-09-17T23:22:48Z" message: 'Created pod: switch-graph-b6ba0-69bcc565d7-ct95z' metadata: creationTimestamp: "2026-09-17T23:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:48Z" name: switch-graph-b6ba0-69bcc565d7.18d63f2e0bf0e48f namespace: kserve-ci-e2e-test resourceVersion: "13112" uid: 32ca2fad-cf83-45a6-b688-70026223d3e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b6ba0 namespace: kserve-ci-e2e-test resourceVersion: "13106" uid: b2b3dedc-1529-4f72-88bd-17e453a7f579 kind: Event lastTimestamp: "2026-09-17T23:22:48Z" message: Scaled up replica set switch-graph-b6ba0-69bcc565d7 from 0 to 1 metadata: creationTimestamp: "2026-09-17T23:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T23:22:48Z" name: switch-graph-b6ba0.18d63f2e0aef3c93 namespace: kserve-ci-e2e-test resourceVersion: "13109" uid: 69cf4dec-9d68-4b12-bfb4-d3af949fa1c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T23:22:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b6ba0 namespace: kserve-ci-e2e-test resourceVersion: "13122" uid: b399bfc0-9dd5-46d7-a692-2fe4674d491e kind: Event lastTimestamp: "2026-09-17T23:22:49Z" message: InferenceGraph [switch-graph-b6ba0] is Ready metadata: creationTimestamp: "2026-09-17T23:22:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T23:22:49Z" name: switch-graph-b6ba0.18d63f2e11253e33 namespace: kserve-ci-e2e-test resourceVersion: "13124" uid: b57246a4-4267-44f4-9d7d-163316399bf1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "26067"