--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:54:06.840269Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-bab33-85475f65fb-t945p to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:54:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:54:06Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e49889adf12c namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: 96b05f2f-d697-49eb-be27-00722522cec3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:07Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: "2026-04-16T16:54:07Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:54:07Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e498a30f4acd namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: eb3c34d9-f57c-43aa-8a60-a8dc9e16acf4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bab33} kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: "2026-04-16T16:54:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:07Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e498a45e6e6b namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 7194005c-fc33-4e7d-a896-ee77df4d9da1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bab33} kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: "2026-04-16T16:54:07Z" message: 'Created container: ensemble-graph-bab33' metadata: creationTimestamp: "2026-04-16T16:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:07Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e498a7cf4f9a namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: 18c1c1be-5db1-444b-874c-183030b15aaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bab33} kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: "2026-04-16T16:54:07Z" message: Started container ensemble-graph-bab33 metadata: creationTimestamp: "2026-04-16T16:54:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:07Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e498a84507d6 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: 54404a44-c255-4d81-bb48-e5ab2dcd85c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bab33} kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: "2026-04-16T16:54:16Z" message: Stopping container ensemble-graph-bab33 metadata: creationTimestamp: "2026-04-16T16:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:16Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e49ae5313742 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 53fdbd4a-b933-496b-970f-295fcb2cccfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bab33} kind: Pod name: ensemble-graph-bab33-85475f65fb-t945p namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 7a7d8f44-59d2-49fb-bdcf-81d13e311d36 kind: Event lastTimestamp: "2026-04-16T16:54:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:43Z" name: ensemble-graph-bab33-85475f65fb-t945p.18a6e49b4b4adcc1 namespace: kserve-ci-e2e-test resourceVersion: "22370" uid: 5baa7dfe-5080-4336-8e2a-ed1234cd9957 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-bab33-85475f65fb namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: 31eb8bca-cdd4-4f11-a061-c52670d47a58 kind: Event lastTimestamp: "2026-04-16T16:54:06Z" message: 'Created pod: ensemble-graph-bab33-85475f65fb-t945p' metadata: creationTimestamp: "2026-04-16T16:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:06Z" name: ensemble-graph-bab33-85475f65fb.18a6e498894d9c59 namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: 5fec499f-9644-4c6c-99ab-48b4adb77059 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 7511b0f6-be38-4b0a-a500-aa13f86df5fb kind: Event lastTimestamp: "2026-04-16T16:54:06Z" message: Scaled up replica set ensemble-graph-bab33-85475f65fb from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:06Z" name: ensemble-graph-bab33.18a6e498884b6fcd namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: 704c6db6-0677-4bd3-94af-e49513f3fd70 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: f69c2adc-d658-42ae-b8d9-a8e51413d820 kind: Event lastTimestamp: "2026-04-16T16:54:06Z" message: InferenceGraph [ensemble-graph-bab33] is Ready metadata: creationTimestamp: "2026-04-16T16:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:06Z" name: ensemble-graph-bab33.18a6e4988c3d475a namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: 5b0e67d1-34c7-484f-9fbd-513f71948357 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:55:27.189330Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22892" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-da571-57db577d67-bk692 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e4ab3edb51d2 namespace: kserve-ci-e2e-test resourceVersion: "22897" uid: c2915841-e9de-4522-9fee-71412fdaf880 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22901" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e4ab58266f08 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: f4092f42-f8ca-4d8c-88d6-7389f79004cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da571} kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e4ab59a81f7a namespace: kserve-ci-e2e-test resourceVersion: "22913" uid: 7a7c96ab-3948-4866-b3c1-15e4dc7ef3bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da571} kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: 'Created container: ensemble-graph-da571' metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e4ab5c855f77 namespace: kserve-ci-e2e-test resourceVersion: "22914" uid: 98a8ac78-5fb0-4525-9693-9905fa55fc77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da571} kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: Started container ensemble-graph-da571 metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e4ab5cfa47d5 namespace: kserve-ci-e2e-test resourceVersion: "22916" uid: 13a9b4d5-93d4-45f0-8bef-67d32d3f6f25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da571} kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: "2026-04-16T17:03:41Z" message: Stopping container ensemble-graph-da571 metadata: creationTimestamp: "2026-04-16T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:41Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e51e69f1db80 namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: e1b58e7a-7046-49da-ad08-d6ccc7652900 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:03:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da571} kind: Pod name: ensemble-graph-da571-57db577d67-bk692 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: b4ec0da9-f3f0-435c-8c18-5f09f47b3b70 kind: Event lastTimestamp: "2026-04-16T17:04:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:03:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:08Z" name: ensemble-graph-da571-57db577d67-bk692.18a6e51ee88a763d namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: 0ad0a0ad-cc35-445f-a933-366f9f49f940 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-da571-57db577d67 namespace: kserve-ci-e2e-test resourceVersion: "22887" uid: eb9457fe-f681-483b-a4ba-626c7791d0e1 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: 'Created pod: ensemble-graph-da571-57db577d67-bk692' metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571-57db577d67.18a6e4ab3e56c2ec namespace: kserve-ci-e2e-test resourceVersion: "22894" uid: 385432a7-662e-49b0-a4e5-720c141f1964 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-da571 namespace: kserve-ci-e2e-test resourceVersion: "22885" uid: 0e77831e-2eb2-443d-92d4-6ec8706057c5 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: Scaled up replica set ensemble-graph-da571-57db577d67 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571.18a6e4ab3d3b22db namespace: kserve-ci-e2e-test resourceVersion: "22888" uid: 1b4526fb-57e1-4214-9528-c5a625c453f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-da571 namespace: kserve-ci-e2e-test resourceVersion: "22905" uid: c4a049fe-0332-4ad4-891d-366cff40eee7 kind: Event lastTimestamp: "2026-04-16T16:55:27Z" message: InferenceGraph [ensemble-graph-da571] is Ready metadata: creationTimestamp: "2026-04-16T16:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:27Z" name: ensemble-graph-da571.18a6e4ab42e7f053 namespace: kserve-ci-e2e-test resourceVersion: "22907" uid: 28fa9ff9-9fe1-46ad-bf14-220327fb8ac5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-da571 namespace: kserve-ci-e2e-test resourceVersion: "22886" uid: c4790664-5a8c-473a-bc89-fc20067c5a64 kind: Event lastTimestamp: "2026-04-16T16:55:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:42Z" name: ensemble-graph-da571.18a6e4aebbcc5e3c namespace: kserve-ci-e2e-test resourceVersion: "23034" uid: 7bd3954d-b3cf-4f40-bd03-fe737d25fe05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-da571 namespace: kserve-ci-e2e-test resourceVersion: "22886" uid: c4790664-5a8c-473a-bc89-fc20067c5a64 kind: Event lastTimestamp: "2026-04-16T16:55:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:42Z" name: ensemble-graph-da571.18a6e4aebc364bb0 namespace: kserve-ci-e2e-test resourceVersion: "23035" uid: 8ab0b89f-324f-42db-aaab-31c022c3c607 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-da571 namespace: kserve-ci-e2e-test resourceVersion: "23033" uid: c4790664-5a8c-473a-bc89-fc20067c5a64 kind: Event lastTimestamp: "2026-04-16T16:55:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:57Z" name: ensemble-graph-da571.18a6e4b23b0c8565 namespace: kserve-ci-e2e-test resourceVersion: "23162" uid: b6cb8ce2-e468-44d4-b94c-3e989f093f9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-da571 namespace: kserve-ci-e2e-test resourceVersion: "23033" uid: c4790664-5a8c-473a-bc89-fc20067c5a64 kind: Event lastTimestamp: "2026-04-16T16:55:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:57Z" name: ensemble-graph-da571.18a6e4b23ba90f5a namespace: kserve-ci-e2e-test resourceVersion: "23165" uid: 6faa33b7-56f2-4c1b-9fd0-7ae939ab06f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:53:31.791568Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e490609d28c7 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: fc3e7159-36c0-49c6-9db7-f5fd57872608 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e49067dac3e4 namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 036f1e7c-d732-49cd-9993-133e9594abdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e4906914d513 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: e92d9c76-ac01-4632-a386-e901d68f287a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: "2026-04-16T16:53:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:32Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e4906f8259a5 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: d7995e4e-1590-4d6f-877a-ea50d292e611 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: "2026-04-16T16:53:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:32Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e4906ff954e0 namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: 79851057-76cb-4da8-819e-84b25bfe372e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: "2026-04-16T16:54:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:53Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e4909f737050 namespace: kserve-ci-e2e-test resourceVersion: "22590" uid: 2e0aad5a-245e-4792-926e-81f428cede2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 3fb00e18-8ad3-425d-bbf2-46722da02dd7 kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv.18a6e4a30ea3d0d1 namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: a7719fc0-cedf-4d8a-a898-79f97ea8e9cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-04ea1-predictor-69944bdbf6 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 38f807af-5e7e-4b37-8767-1cb7f99ea284 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'Created pod: error-404-isvc-04ea1-predictor-69944bdbf6-kr8wv' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1-predictor-69944bdbf6.18a6e490602842f7 namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: cecf1289-691f-4cf9-9191-124c4a924c02 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 0b2e5e68-e147-4e2a-b141-8b800f2f37f9 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Scaled up replica set error-404-isvc-04ea1-predictor-69944bdbf6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1-predictor.18a6e4905e99d19b namespace: kserve-ci-e2e-test resourceVersion: "21510" uid: cc683253-40d9-4ea3-96ea-80a14bbf272e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: f9da62d9-c47d-4959-8efd-d6729e205c5c kind: Event lastTimestamp: "2026-04-16T16:53:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:46Z" name: error-404-isvc-04ea1-predictor.18a6e493dd67e6eb namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: dc5b82ea-54bd-4fc0-9df6-df6c65f1ba69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: f9da62d9-c47d-4959-8efd-d6729e205c5c kind: Event lastTimestamp: "2026-04-16T16:53:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:46Z" name: error-404-isvc-04ea1-predictor.18a6e493ddda99aa namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: b960c35c-66f6-4a8c-aedb-00102aab4ced reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:54:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: f9da62d9-c47d-4959-8efd-d6729e205c5c kind: Event lastTimestamp: "2026-04-16T16:54:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:46Z" name: error-404-isvc-04ea1-predictor.18a6e4975cd8a5f8 namespace: kserve-ci-e2e-test resourceVersion: "22394" uid: 529a6dba-a5a3-4a52-8bb6-09e706315285 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:54:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: f9da62d9-c47d-4959-8efd-d6729e205c5c kind: Event lastTimestamp: "2026-04-16T16:54:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:46Z" name: error-404-isvc-04ea1-predictor.18a6e4975d4b2949 namespace: kserve-ci-e2e-test resourceVersion: "22395" uid: 6a5f773c-107c-44db-b559-67356c325606 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 586946ed-bc15-4d01-a473-e79d3dc171e1 kind: Event lastTimestamp: "2026-04-16T16:54:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:23Z" name: error-404-isvc-04ea1.18a6e4905eaf22b4 namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: c248a554-66c8-4424-8cca-7ec89e00360e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 586946ed-bc15-4d01-a473-e79d3dc171e1 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-04ea1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-04ea1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1.18a6e4905f313f49 namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: cbe40fc0-c2a7-4967-983e-d04edbe1d9bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 586946ed-bc15-4d01-a473-e79d3dc171e1 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-04ea1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-04ea1.18a6e4905f319739 namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: 74ca6cd7-811e-48a1-9b97-4ac2ccb9994c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "22175" uid: 586946ed-bc15-4d01-a473-e79d3dc171e1 kind: Event lastTimestamp: "2026-04-16T16:54:23Z" message: InferenceService [error-404-isvc-04ea1] is Ready metadata: creationTimestamp: "2026-04-16T16:54:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:23Z" name: error-404-isvc-04ea1.18a6e49c83a3e602 namespace: kserve-ci-e2e-test resourceVersion: "22176" uid: ea99ad61-b83c-47f5-b789-f58f19d6a13c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:44:06.715510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-39f4a-predictor-8fc685496-rwzlb to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e40ccf74c1ad namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 2083b870-9edf-47d3-bedb-7b8befd60329 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e40cd72e70c5 namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 080d5c5a-fc75-420c-a998-73d3f72ce35c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e40cd881021e namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: 750e75e7-2ead-433d-b70a-3412fc0ee38f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e40cdce8e73e namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 558fc3c0-eb12-4be5-af5d-4d55d2294cc8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e40cdd5584c5 namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 61bfbc31-bcee-4f94-bdfd-8a1cfe03ce62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:44:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: "2026-04-16T16:44:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:44:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:48Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e40cf1e153e1 namespace: kserve-ci-e2e-test resourceVersion: "17627" uid: 17c8033c-58b2-4f82-a0c1-a9c66b803077 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: f842b0b3-f94c-4fa7-8d35-4f3ffdbd8cc5 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: error-404-isvc-39f4a-predictor-8fc685496-rwzlb.18a6e4905f5e8837 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 68f5553b-e1e7-4c9c-87a0-6c7a4d35a619 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-39f4a-predictor-8fc685496 namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: 69749b2d-657a-4553-99b8-4a4973a35139 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: 'Created pod: error-404-isvc-39f4a-predictor-8fc685496-rwzlb' metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor-8fc685496.18a6e40ccf034ef2 namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: cd1cada5-8760-4419-97d9-a5b326d2716a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 0585eacc-4966-4b0b-8a34-be83dce7b90e kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Scaled up replica set error-404-isvc-39f4a-predictor-8fc685496 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a-predictor.18a6e40ccd466e5f namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: 109d1dfd-9af3-45ac-aebc-25aa30ba826a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: b4b202b1-639a-4ed4-ba8e-51d254f458d7 kind: Event lastTimestamp: "2026-04-16T16:44:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:21Z" name: error-404-isvc-39f4a-predictor.18a6e4104bc82a5f namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: 55fc5123-bd83-430c-afeb-26e31aec5937 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: b4b202b1-639a-4ed4-ba8e-51d254f458d7 kind: Event lastTimestamp: "2026-04-16T16:44:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:21Z" name: error-404-isvc-39f4a-predictor.18a6e4104c3b4eca namespace: kserve-ci-e2e-test resourceVersion: "17368" uid: bd4d4f11-a313-44df-b885-3ccc33863d83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: b4b202b1-639a-4ed4-ba8e-51d254f458d7 kind: Event lastTimestamp: "2026-04-16T16:45:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:36Z" name: error-404-isvc-39f4a-predictor.18a6e413cb1ef1ae namespace: kserve-ci-e2e-test resourceVersion: "18051" uid: 33f2080e-7341-4235-b907-a87096289a33 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: b4b202b1-639a-4ed4-ba8e-51d254f458d7 kind: Event lastTimestamp: "2026-04-16T16:45:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:36Z" name: error-404-isvc-39f4a-predictor.18a6e413cb704884 namespace: kserve-ci-e2e-test resourceVersion: "18052" uid: 03acdc02-89f4-4436-a362-62537a9b2a8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: 94da8187-43bb-4675-a6b0-542c88dd3b57 kind: Event lastTimestamp: "2026-04-16T16:44:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:58Z" name: error-404-isvc-39f4a.18a6e40ccd048336 namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: aaa1e8da-e75d-4876-9ba9-6e887f92436e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: 94da8187-43bb-4675-a6b0-542c88dd3b57 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-39f4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-39f4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a.18a6e40ccd755e61 namespace: kserve-ci-e2e-test resourceVersion: "17196" uid: 5335c670-0450-4cd9-8068-8b71f5d8ea0b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: 94da8187-43bb-4675-a6b0-542c88dd3b57 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-39f4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:06Z" name: error-404-isvc-39f4a.18a6e40ccd75acdf namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: d4827b6d-c78a-4e52-9ef6-4f04629ba5fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17717" uid: 94da8187-43bb-4675-a6b0-542c88dd3b57 kind: Event lastTimestamp: "2026-04-16T16:44:58Z" message: InferenceService [error-404-isvc-39f4a] is Ready metadata: creationTimestamp: "2026-04-16T16:44:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:58Z" name: error-404-isvc-39f4a.18a6e418d6128053 namespace: kserve-ci-e2e-test resourceVersion: "17719" uid: 97c2e408-8f0b-4ea3-b368-edc6d815c50c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:05:02.416264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27730" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-45e11-predictor-5c496f76fd-sc45w to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e5312d0dc6a1 namespace: kserve-ci-e2e-test resourceVersion: "27735" uid: 01a24c9c-d980-44bb-9660-36fecc4bf5c2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27739" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e531345f576b namespace: kserve-ci-e2e-test resourceVersion: "27750" uid: fda0cc0c-7914-4c1d-a4aa-dff1b63fc523 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27732" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e53135b5b29c namespace: kserve-ci-e2e-test resourceVersion: "27753" uid: 3a805217-256f-4dfb-b59d-1ca9e5b4271a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27732" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e531387ec041 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 270ff48e-be64-4b1b-ae21-20fb16d41127 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27732" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e5313903db14 namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: d179174e-7276-438f-b660-5bbfb81e06f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27732" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: "2026-04-16T17:05:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:43Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e53143187646 namespace: kserve-ci-e2e-test resourceVersion: "28160" uid: 95ead721-dcf8-4b0b-826a-dd593a5908e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w namespace: kserve-ci-e2e-test resourceVersion: "27732" uid: ac6f9720-1c2e-45a3-9966-cfec5f0c30ca kind: Event lastTimestamp: "2026-04-16T17:14:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:14:27Z" name: error-404-isvc-45e11-predictor-5c496f76fd-sc45w.18a6e5b4a875cbfe namespace: kserve-ci-e2e-test resourceVersion: "31836" uid: 3cceee16-0b04-45ea-9ac4-984b3b3d66ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-45e11-predictor-5c496f76fd namespace: kserve-ci-e2e-test resourceVersion: "27722" uid: 5e5ac271-4cd3-4c99-a1c0-a3f6ea8780f6 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'Created pod: error-404-isvc-45e11-predictor-5c496f76fd-sc45w' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor-5c496f76fd.18a6e5312caebd4f namespace: kserve-ci-e2e-test resourceVersion: "27733" uid: 549d5363-7b6b-46a7-a50a-5e58110ea1eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27716" uid: 5097f485-12fb-4779-a46a-f70c2f1bc384 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Scaled up replica set error-404-isvc-45e11-predictor-5c496f76fd from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11-predictor.18a6e5312afc7471 namespace: kserve-ci-e2e-test resourceVersion: "27723" uid: aaf46359-f362-4881-9916-3a33eba2104b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27721" uid: ca7adfb1-91df-4537-a529-032b9d3f5569 kind: Event lastTimestamp: "2026-04-16T17:05:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:17Z" name: error-404-isvc-45e11-predictor.18a6e534a9b24484 namespace: kserve-ci-e2e-test resourceVersion: "27915" uid: a5225e9f-ebca-47ae-b71d-b4f97422b149 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27721" uid: ca7adfb1-91df-4537-a529-032b9d3f5569 kind: Event lastTimestamp: "2026-04-16T17:05:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:17Z" name: error-404-isvc-45e11-predictor.18a6e534aa17f95b namespace: kserve-ci-e2e-test resourceVersion: "27917" uid: b45aa3af-7185-4454-8c32-93fcc66d1817 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27916" uid: ca7adfb1-91df-4537-a529-032b9d3f5569 kind: Event lastTimestamp: "2026-04-16T17:06:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:17Z" name: error-404-isvc-45e11-predictor.18a6e53828d45158 namespace: kserve-ci-e2e-test resourceVersion: "28460" uid: d3cd180c-454e-49e7-8f4e-de4fdbf01d15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27916" uid: ca7adfb1-91df-4537-a529-032b9d3f5569 kind: Event lastTimestamp: "2026-04-16T17:06:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:17Z" name: error-404-isvc-45e11-predictor.18a6e538293c3f7f namespace: kserve-ci-e2e-test resourceVersion: "28461" uid: f77a3f04-e18a-4f12-8d43-5eb2ee7e5caf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "27694" uid: 7bd1f632-ab41-44f4-b2cc-05f946d669ec kind: Event lastTimestamp: "2026-04-16T17:05:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:53Z" name: error-404-isvc-45e11.18a6e5312b05dbd6 namespace: kserve-ci-e2e-test resourceVersion: "28242" uid: bb613c07-1a3c-4f30-99cf-589a97f0a5e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "27694" uid: 7bd1f632-ab41-44f4-b2cc-05f946d669ec kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-45e11": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-45e11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11.18a6e5312baa3d72 namespace: kserve-ci-e2e-test resourceVersion: "27727" uid: c0caf52c-1b2c-49e0-afd5-4347d7042289 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "27694" uid: 7bd1f632-ab41-44f4-b2cc-05f946d669ec kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-45e11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-45e11.18a6e5312baa8ea7 namespace: kserve-ci-e2e-test resourceVersion: "27731" uid: e31e0154-7488-45e4-9b19-1d24108ce4a6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28239" uid: 7bd1f632-ab41-44f4-b2cc-05f946d669ec kind: Event lastTimestamp: "2026-04-16T17:05:53Z" message: InferenceService [error-404-isvc-45e11] is Ready metadata: creationTimestamp: "2026-04-16T17:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:53Z" name: error-404-isvc-45e11.18a6e53d27266d21 namespace: kserve-ci-e2e-test resourceVersion: "28240" uid: 3a10f179-dae5-4e42-9e45-49d927c33cbf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:54:52.097600Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22540" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-64f8d-predictor-594459c94b-lb4vm to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e4a31339efd0 namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: b2f56010-0bc9-4231-8c84-c732a4d40795 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e4a31a9027b5 namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: d502d657-7b7f-4c31-acd2-361385e6aba2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e4a31bdc55b5 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 684f92a0-e35a-4ca8-81db-98a0aa22cebe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e4a31e86e02b namespace: kserve-ci-e2e-test resourceVersion: "22563" uid: 54bb0f7f-bc94-4b32-b9e4-578e6040063e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e4a31ef87dc7 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: 9b22b246-61ae-4bb7-8340-d593c94ea78a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: "2026-04-16T16:55:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:55:34Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e4a34d942a64 namespace: kserve-ci-e2e-test resourceVersion: "22975" uid: 4c177a91-118b-4371-9928-1d53f15dec99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 0e5e949e-dee6-4211-b47b-6d48f6d2876b kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:16Z" name: error-404-isvc-64f8d-predictor-594459c94b-lb4vm.18a6e52697e602c6 namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: dcc44556-d115-4490-9a78-41bff71bd68d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-64f8d-predictor-594459c94b namespace: kserve-ci-e2e-test resourceVersion: "22530" uid: 71c7c6aa-d6b0-4a48-be51-5e3efe8e52b3 kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: 'Created pod: error-404-isvc-64f8d-predictor-594459c94b-lb4vm' metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor-594459c94b.18a6e4a312a7c8dd namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: c24395a7-642c-44a7-99f2-403a29ee8411 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22527" uid: 788ce63b-5d5c-4056-91f6-397588c23d8a kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Scaled up replica set error-404-isvc-64f8d-predictor-594459c94b from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d-predictor.18a6e4a310476597 namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: a3eb89f7-105d-4f2d-bb80-8a8841184928 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:55:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 46198619-448c-4263-87a6-70c58717f794 kind: Event lastTimestamp: "2026-04-16T16:55:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:37Z" name: error-404-isvc-64f8d-predictor.18a6e4a68ea95c8c namespace: kserve-ci-e2e-test resourceVersion: "22999" uid: 56c352fd-22a2-48e4-af00-b4388ed1f0dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:55:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 46198619-448c-4263-87a6-70c58717f794 kind: Event lastTimestamp: "2026-04-16T16:55:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:37Z" name: error-404-isvc-64f8d-predictor.18a6e4a68f11d7b8 namespace: kserve-ci-e2e-test resourceVersion: "23000" uid: 946ee25d-317f-4faf-a6f3-b3f61feac5db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:55:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22704" uid: 46198619-448c-4263-87a6-70c58717f794 kind: Event lastTimestamp: "2026-04-16T16:56:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:07Z" name: error-404-isvc-64f8d-predictor.18a6e4b10befed99 namespace: kserve-ci-e2e-test resourceVersion: "23281" uid: 6d8bb167-6764-4f78-b425-c3f89c146a4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:55:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22704" uid: 46198619-448c-4263-87a6-70c58717f794 kind: Event lastTimestamp: "2026-04-16T16:56:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:07Z" name: error-404-isvc-64f8d-predictor.18a6e4b10cbc8366 namespace: kserve-ci-e2e-test resourceVersion: "23282" uid: cf9c3aa9-d7ad-4c41-9749-50a51e60b2dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "22500" uid: ec190c61-19de-46c8-97dc-a14232659a00 kind: Event lastTimestamp: "2026-04-16T16:55:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:44Z" name: error-404-isvc-64f8d.18a6e4a30fd2a59f namespace: kserve-ci-e2e-test resourceVersion: "23071" uid: 7466206a-1dda-4269-8d6a-854560b4d6af reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "22500" uid: ec190c61-19de-46c8-97dc-a14232659a00 kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-64f8d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-64f8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d.18a6e4a3107fa3a0 namespace: kserve-ci-e2e-test resourceVersion: "22534" uid: b5c4a302-782b-4882-8469-afe9f8b0aa97 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "22500" uid: ec190c61-19de-46c8-97dc-a14232659a00 kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-64f8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:52Z" name: error-404-isvc-64f8d.18a6e4a31080220d namespace: kserve-ci-e2e-test resourceVersion: "22536" uid: 3e873b1c-7742-45fd-979e-11f2df76f534 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23069" uid: ec190c61-19de-46c8-97dc-a14232659a00 kind: Event lastTimestamp: "2026-04-16T16:55:44Z" message: InferenceService [error-404-isvc-64f8d] is Ready metadata: creationTimestamp: "2026-04-16T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:44Z" name: error-404-isvc-64f8d.18a6e4af3212a005 namespace: kserve-ci-e2e-test resourceVersion: "23070" uid: 857cd289-9ca6-4395-b726-02f899ad8e6a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:41:57.034371Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9c599-predictor-64946668d-nbsr6 to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:41:57Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3ee9ddbbaa6 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 12dbb40e-e2ab-4cc5-9476-9477f54ef3b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:41:57Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:41:57Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3eea49b609a namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 2a1a2908-9993-4e5b-9c16-f20f3724ac1f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:41:57Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:41:57Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3eeadad85cf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 364d3e13-595e-463d-9fa2-167e805a9840 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" in 13.944s (13.944s including waiting). Image size: 1331783784 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3f1ecd66bd5 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 59b784f8-fbac-4a22-88af-7f502e83e92b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3f1f2b4cedd namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: c439ac5b-c4c9-4b1d-844c-86a5164f2663 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3f1f353dac7 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: cd7ecfcb-d40b-4f31-8528-a450b30efe26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e3f2159924d3 namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: dd88e380-071f-4c01-99d6-b70923790f1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c599-predictor-64946668d-nbsr6 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 84927f2c-3dcf-4a35-9cdd-6f8b51ed08d7 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:31Z" name: error-404-isvc-9c599-predictor-64946668d-nbsr6.18a6e404b4558922 namespace: kserve-ci-e2e-test resourceVersion: "16699" uid: 2473f863-1eca-4429-963c-5292e9700558 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9c599-predictor-64946668d namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 57e8e0cb-2b03-48cd-a067-4c44a74d762a kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: 'Created pod: error-404-isvc-9c599-predictor-64946668d-nbsr6' metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: error-404-isvc-9c599-predictor-64946668d.18a6e3ee9b7706cf namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 2c7a65d6-75f5-44cf-96ca-40b1362de1bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 22b9dcea-04b3-403b-9c12-9335c22d9fdb kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Scaled up replica set error-404-isvc-9c599-predictor-64946668d from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: error-404-isvc-9c599-predictor.18a6e3ee985ffee4 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: 837bcb52-1690-4012-b9fb-146696050bf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: f5cfa7e9-4f9d-4cf5-bfbf-0152061def15 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:11Z" name: error-404-isvc-9c599-predictor.18a6e3f217492245 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 488b2a26-6a0c-47a0-9ede-c672f0131ba8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: f5cfa7e9-4f9d-4cf5-bfbf-0152061def15 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:11Z" name: error-404-isvc-9c599-predictor.18a6e3f217b75d32 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: a30e93ca-18fe-424e-b9cd-4d5e5308ba55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:42:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: f5cfa7e9-4f9d-4cf5-bfbf-0152061def15 kind: Event lastTimestamp: "2026-04-16T16:43:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:27Z" name: error-404-isvc-9c599-predictor.18a6e3f59681e038 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 5ff9542b-5cd8-41f1-93b3-b9c5d2c8f206 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:42:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: f5cfa7e9-4f9d-4cf5-bfbf-0152061def15 kind: Event lastTimestamp: "2026-04-16T16:43:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:27Z" name: error-404-isvc-9c599-predictor.18a6e3f5970fd9fe namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: b538191e-a654-41bc-9d74-fa30309afc8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c599 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 55ca0df8-a630-4b31-9ea3-3c1222cdea00 kind: Event lastTimestamp: "2026-04-16T16:43:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:13Z" name: error-404-isvc-9c599.18a6e3ee98cf18bb namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 8e2deeb2-5aff-4711-b705-4ced704859e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c599 namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 55ca0df8-a630-4b31-9ea3-3c1222cdea00 kind: Event lastTimestamp: "2026-04-16T16:43:13Z" message: InferenceService [error-404-isvc-9c599] is Ready metadata: creationTimestamp: "2026-04-16T16:43:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:13Z" name: error-404-isvc-9c599.18a6e4004e5867a8 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 5a5d1710-81ff-4ed8-afe6-eb20b4344b20 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:04:16.992085Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:04:16Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e526998fcd44 namespace: kserve-ci-e2e-test resourceVersion: "27185" uid: 37cbc365-fc1a-4c7f-bedc-2fffefa82029 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27191" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:04:17Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e526a0c55d0b namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: 7575d2fb-ca6b-4cbc-928b-62b387eae3e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:17Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e526a206036b namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: ae8baaf2-f3fd-41f2-93bb-cf3f624684e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:17Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e526a88281df namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: 04880495-bc8c-44f9-aa92-a53236ba5d83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:17Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e526a90a96d4 namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 8c4e616f-4a84-49f2-b00d-27a5cf884175 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: "2026-04-16T17:21:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:21:48Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e526c13de001 namespace: kserve-ci-e2e-test resourceVersion: "34925" uid: e2376322-be6f-4a1d-89fa-c05a0c3d0482 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 0bd3d6ac-f4fe-4a34-9eef-e99722e2e59b kind: Event lastTimestamp: "2026-04-16T17:21:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:21:46Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h.18a6e61b0023c303 namespace: kserve-ci-e2e-test resourceVersion: "34879" uid: fd927a46-c763-4c35-b973-09454cb04a9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b9f91-predictor-cb47fbcb4 namespace: kserve-ci-e2e-test resourceVersion: "27167" uid: a577eb72-231e-4179-b458-6b04b6074c8c kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: 'Created pod: error-404-isvc-b9f91-predictor-cb47fbcb4-rhs7h' metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:16Z" name: error-404-isvc-b9f91-predictor-cb47fbcb4.18a6e52698e632f2 namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: f9ef7324-afec-43ad-bb79-76767cc9aef0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: a149f0a2-b350-49e8-af2b-ae7dcd41aa6c kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: Scaled up replica set error-404-isvc-b9f91-predictor-cb47fbcb4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:16Z" name: error-404-isvc-b9f91-predictor.18a6e526972dacb1 namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: b58f2cb6-d862-4310-a96a-5ed038948231 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27170" uid: 4f469bab-02f2-4bd1-9646-7caa57ba57f3 kind: Event lastTimestamp: "2026-04-16T17:04:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:31Z" name: error-404-isvc-b9f91-predictor.18a6e52a1625c665 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: 00efe625-9eba-4b8b-8aee-f58af786d58d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27170" uid: 4f469bab-02f2-4bd1-9646-7caa57ba57f3 kind: Event lastTimestamp: "2026-04-16T17:04:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:31Z" name: error-404-isvc-b9f91-predictor.18a6e52a16893648 namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: 9023586f-65c6-4120-825f-02c71989e603 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:04:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: 4f469bab-02f2-4bd1-9646-7caa57ba57f3 kind: Event lastTimestamp: "2026-04-16T17:05:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:32Z" name: error-404-isvc-b9f91-predictor.18a6e52d954359e8 namespace: kserve-ci-e2e-test resourceVersion: "28056" uid: 2077f7ca-eb03-4e25-87d9-bf4c6f46776b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:04:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: 4f469bab-02f2-4bd1-9646-7caa57ba57f3 kind: Event lastTimestamp: "2026-04-16T17:05:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:32Z" name: error-404-isvc-b9f91-predictor.18a6e52d95a94366 namespace: kserve-ci-e2e-test resourceVersion: "28057" uid: 231d9c84-84df-4227-9f15-3c5c0e51da1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: eeec1af8-817a-4656-8061-f599f768311d kind: Event lastTimestamp: "2026-04-16T17:05:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:08Z" name: error-404-isvc-b9f91.18a6e52697695ae8 namespace: kserve-ci-e2e-test resourceVersion: "27837" uid: 075910fd-922b-42b7-814c-5b4721f54c26 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: eeec1af8-817a-4656-8061-f599f768311d kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-b9f91": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b9f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:16Z" name: error-404-isvc-b9f91.18a6e526980af0e2 namespace: kserve-ci-e2e-test resourceVersion: "27179" uid: a10b323a-0280-4577-8d73-a7ec02ba2cb5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: eeec1af8-817a-4656-8061-f599f768311d kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b9f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:16Z" name: error-404-isvc-b9f91.18a6e526980b4505 namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: 3be832e7-b935-4133-8f82-b4cafd055b46 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27831" uid: eeec1af8-817a-4656-8061-f599f768311d kind: Event lastTimestamp: "2026-04-16T17:05:08Z" message: InferenceService [error-404-isvc-b9f91] is Ready metadata: creationTimestamp: "2026-04-16T17:05:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:08Z" name: error-404-isvc-b9f91.18a6e532a5335132 namespace: kserve-ci-e2e-test resourceVersion: "27832" uid: 590a8c60-7c5c-493b-be91-94209413433d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:52:57.054566Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bab33-predictor-86bd4f56b4-97wxn to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e4884a2089f1 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 817e0ff6-3032-41ac-9c7d-0432e1e0b345 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e48851053839 namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 9db5708f-ab50-4a75-8009-8202ada97ccd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e488523939cc namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: f0d6438e-cbec-44a6-bc30-8b29ba799d4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e4885561ddd3 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: fa5f2030-2ca6-46ec-89b2-a51537a3ff26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e48855c1ffaa namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: 04177969-d0e2-4a40-b2d7-1a5b067a59c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: "2026-04-16T16:54:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:18Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e48872f258cc namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: c8e18d57-c09f-4a8c-a229-2d81900a5c0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 7004b4b1-b124-47f6-9777-6d7ea8068d1e kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn.18a6e49aff9b1ea8 namespace: kserve-ci-e2e-test resourceVersion: "22067" uid: 29c1f933-6ec7-40d7-a6b8-f07dec53b2f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bab33-predictor-86bd4f56b4 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 24be4b6c-32e0-4534-9ff3-208f64f5ba12 kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: 'Created pod: error-404-isvc-bab33-predictor-86bd4f56b4-97wxn' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33-predictor-86bd4f56b4.18a6e48848f0367d namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: d9af28e5-3375-4b59-b80f-20e0f35c83c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bab33-predictor namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 6385e03f-123e-4b4f-9a66-d15e4bf10ec8 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: Scaled up replica set error-404-isvc-bab33-predictor-86bd4f56b4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:52:56Z" name: error-404-isvc-bab33-predictor.18a6e48846adebae namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 52f0fe64-7431-411b-9639-a2e3ed853cf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:53:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bab33-predictor namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 37cb7478-e984-4e0b-bfe5-68d7684828bc kind: Event lastTimestamp: "2026-04-16T16:54:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:53:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:12Z" name: error-404-isvc-bab33-predictor.18a6e48bc585db84 namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 9b0d84c1-6615-4e10-bc07-32381095557d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:53:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bab33-predictor namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 37cb7478-e984-4e0b-bfe5-68d7684828bc kind: Event lastTimestamp: "2026-04-16T16:54:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:53:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:12Z" name: error-404-isvc-bab33-predictor.18a6e48bc5ec20a2 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: cd9f746b-50b8-4be4-948c-1b001aa5560c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: e472ff71-ef61-475b-921a-d114e8129d1f kind: Event lastTimestamp: "2026-04-16T16:53:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:48Z" name: error-404-isvc-bab33.18a6e48846df4b31 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 025e2b8a-1f0a-4f54-8355-b642eea3dcce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: e472ff71-ef61-475b-921a-d114e8129d1f kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-bab33": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bab33": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33.18a6e48847cbf36b namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 01a9fa5e-ab65-4fdc-8f22-5e4006ca6ea6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: e472ff71-ef61-475b-921a-d114e8129d1f kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bab33": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:52:57Z" name: error-404-isvc-bab33.18a6e48847cc3373 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 64660466-408f-4878-bd3f-0b7da7a7cc12 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: e472ff71-ef61-475b-921a-d114e8129d1f kind: Event lastTimestamp: "2026-04-16T16:53:48Z" message: InferenceService [error-404-isvc-bab33] is Ready metadata: creationTimestamp: "2026-04-16T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:48Z" name: error-404-isvc-bab33.18a6e494585e9861 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 44bebec5-47d8-4f4b-8e1b-59581742948d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:54:17.422395Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-da571-predictor-586f5c8454-pfzr8 to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e49b006c63b9 namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: aff8fd12-1a83-4d89-b93e-1c9795d882b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e49b083d0159 namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: fe47e66a-ac14-419f-8dbe-1bf555595033 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e49b099632f7 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 1b3eb844-860a-4ccc-9a30-cbde0202c4de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e49b0da01414 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 4d72f999-a06e-45fb-97de-a2311890f649 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e49b0e13bd7a namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: dfc35e82-ef26-4c3d-bedc-5514674edb16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: "2026-04-16T16:54:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:58Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e49b2176dd57 namespace: kserve-ci-e2e-test resourceVersion: "22631" uid: c4ec910a-3b4a-4b23-9be6-c1bad5fef7d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da571-predictor-586f5c8454-pfzr8 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: e8ce8d3e-23ce-46ec-983b-495828ebd970 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-da571-predictor-586f5c8454-pfzr8.18a6e51e75da830e namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 4f2b1740-07fc-4501-9290-661565edc9a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-da571-predictor-586f5c8454 namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: 335dc773-392d-4a16-bca4-7ee3fd701ad5 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'Created pod: error-404-isvc-da571-predictor-586f5c8454-pfzr8' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor-586f5c8454.18a6e49affbb7eba namespace: kserve-ci-e2e-test resourceVersion: "22068" uid: f7d8b7d0-fc22-4351-a537-9287d2610b1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: e2e7250f-c23e-44a4-be7d-4c241e2f5371 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Scaled up replica set error-404-isvc-da571-predictor-586f5c8454 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571-predictor.18a6e49afdaeb307 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: 4ce15cc9-4ccd-4a17-b813-0abaee4acf8d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: 8e3ccd8b-d5a6-4b77-9947-c04026123992 kind: Event lastTimestamp: "2026-04-16T16:54:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:32Z" name: error-404-isvc-da571-predictor.18a6e49e7cfa0505 namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: 512c2196-085d-415e-9f33-9e3fdd64697b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22054" uid: 8e3ccd8b-d5a6-4b77-9947-c04026123992 kind: Event lastTimestamp: "2026-04-16T16:54:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:32Z" name: error-404-isvc-da571-predictor.18a6e49e7d65965d namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: 5a753423-7d97-45b1-b991-a967c6e60ce3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 8e3ccd8b-d5a6-4b77-9947-c04026123992 kind: Event lastTimestamp: "2026-04-16T16:55:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:32Z" name: error-404-isvc-da571-predictor.18a6e4a1fc3d687e namespace: kserve-ci-e2e-test resourceVersion: "22955" uid: d808312c-2bea-4554-b91b-ea27af8a841d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:54:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 8e3ccd8b-d5a6-4b77-9947-c04026123992 kind: Event lastTimestamp: "2026-04-16T16:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:32Z" name: error-404-isvc-da571-predictor.18a6e4a1fcea010f namespace: kserve-ci-e2e-test resourceVersion: "22956" uid: 3382854a-642b-41cd-a975-ce83612f8313 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: e1a942da-9fae-479c-8936-c503ecc9255e kind: Event lastTimestamp: "2026-04-16T16:55:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:09Z" name: error-404-isvc-da571.18a6e49afe45cc4b namespace: kserve-ci-e2e-test resourceVersion: "22737" uid: c623aedb-e096-4c3c-8a08-f61952cfad2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: e1a942da-9fae-479c-8936-c503ecc9255e kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-da571": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-da571": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571.18a6e49afedcce81 namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 26b54948-bc1b-428a-a4c2-09893f4e284b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: e1a942da-9fae-479c-8936-c503ecc9255e kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-da571": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:17Z" name: error-404-isvc-da571.18a6e49afedda048 namespace: kserve-ci-e2e-test resourceVersion: "22066" uid: b7231aba-f756-46a1-91ed-51b422a5ebc9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "22735" uid: e1a942da-9fae-479c-8936-c503ecc9255e kind: Event lastTimestamp: "2026-04-16T16:55:09Z" message: InferenceService [error-404-isvc-da571] is Ready metadata: creationTimestamp: "2026-04-16T16:55:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:09Z" name: error-404-isvc-da571.18a6e4a7065aef46 namespace: kserve-ci-e2e-test resourceVersion: "22736" uid: 9ef801f6-b733-472e-bb23-66810828feba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:43:31.927878Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-df178-predictor-58dc95b6f4-rjs9z to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:43:31Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e404b5f3a32c namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: e133d8b5-ad84-4c19-869b-f76030961ced reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:43:32Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e404bd52baa3 namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: 194f2910-8bfa-45fd-82d9-0f1bcb370126 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e404beb63c26 namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: f60d0d6d-b6cb-4bf6-b4b3-c74932c7781e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e404c2f52565 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: ed19f842-6329-40bb-884a-c8c0c1c09c05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e404c35e5156 namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 3e45cacf-21ec-47bc-8c33-bedc60815967 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: "2026-04-16T16:44:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:13Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e404c53b7f79 namespace: kserve-ci-e2e-test resourceVersion: "17290" uid: e4f00784-c19b-453a-85c7-4de35109ecec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 691deef6-3160-4e56-bedb-5a4579b66467 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:56Z" name: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z.18a6e488420795e6 namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: da9bce7f-3546-40c1-9462-f73055696963 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-df178-predictor-58dc95b6f4 namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: 9e74df2a-a6f4-48da-b6fd-5b7dee2c5370 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'Created pod: error-404-isvc-df178-predictor-58dc95b6f4-rjs9z' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:31Z" name: error-404-isvc-df178-predictor-58dc95b6f4.18a6e404b55ea3a7 namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 199dc348-bd38-4cb1-b497-a449493f5a2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: 7f98affb-3257-49bb-b579-b2368cdb36d9 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: Scaled up replica set error-404-isvc-df178-predictor-58dc95b6f4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:31Z" name: error-404-isvc-df178-predictor.18a6e404b3b8df99 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: d76aa75f-5284-4bc4-8eeb-d2542b47a330 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16692" uid: 3b71598c-3fe2-4820-89c7-87658e7c59b2 kind: Event lastTimestamp: "2026-04-16T16:44:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:01Z" name: error-404-isvc-df178-predictor.18a6e40832770361 namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: 5eaab07e-ff38-46eb-b5ff-edb4088873ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16692" uid: 3b71598c-3fe2-4820-89c7-87658e7c59b2 kind: Event lastTimestamp: "2026-04-16T16:44:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:01Z" name: error-404-isvc-df178-predictor.18a6e4083304c836 namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: e44c1744-d439-4459-b29f-4e3a356a5423 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 3b71598c-3fe2-4820-89c7-87658e7c59b2 kind: Event lastTimestamp: "2026-04-16T16:45:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:01Z" name: error-404-isvc-df178-predictor.18a6e40f30c42b40 namespace: kserve-ci-e2e-test resourceVersion: "17758" uid: 15843da1-ceb1-4bc0-a9bc-43c93b0fcfd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 3b71598c-3fe2-4820-89c7-87658e7c59b2 kind: Event lastTimestamp: "2026-04-16T16:45:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:01Z" name: error-404-isvc-df178-predictor.18a6e40f312dfb22 namespace: kserve-ci-e2e-test resourceVersion: "17759" uid: adc90a22-db48-438c-a5e4-3394e6df65ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: a5233cc3-9cc5-4962-afdd-dec8157c0ea2 kind: Event lastTimestamp: "2026-04-16T16:44:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:23Z" name: error-404-isvc-df178.18a6e404b3c6e2a8 namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: 704f9952-bdf2-4766-9e5d-b61579877242 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: a5233cc3-9cc5-4962-afdd-dec8157c0ea2 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-df178": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df178": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:31Z" name: error-404-isvc-df178.18a6e404b4700d8c namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 7e5caaa7-b59b-4d0f-bd8f-61dabbc98ec4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: a5233cc3-9cc5-4962-afdd-dec8157c0ea2 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df178": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:31Z" name: error-404-isvc-df178.18a6e404b4705980 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 0de64f3a-9b81-4f1a-b9b0-52965c7018dd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: a5233cc3-9cc5-4962-afdd-dec8157c0ea2 kind: Event lastTimestamp: "2026-04-16T16:44:23Z" message: InferenceService [error-404-isvc-df178] is Ready metadata: creationTimestamp: "2026-04-16T16:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:23Z" name: error-404-isvc-df178.18a6e410a95f8280 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: d1a9566b-008b-4f8d-bf90-18bb3929f0ae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:03:42.130547Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e51e7ba6ec29 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: d007d994-2309-4862-a85f-f25f3f917e3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e51e831b74f3 namespace: kserve-ci-e2e-test resourceVersion: "26804" uid: 01694593-dc38-4ab3-a6da-a7c5d6dda48e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e51e844daea7 namespace: kserve-ci-e2e-test resourceVersion: "26806" uid: fcb69a22-db4f-4578-9f2b-020d42b4c43d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e51e884865a9 namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: 5ef6f5c9-c3b0-4a25-affc-b635a0c7d0e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e51e88c29c89 namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: a5f98f16-34bf-43d3-ab52-a0840c194be1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: "2026-04-16T17:05:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:03Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e51e949800eb namespace: kserve-ci-e2e-test resourceVersion: "27775" uid: 9c0e6f31-a82e-4476-b40a-ae84fc45f806 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: e6ba8407-fa0f-4b26-9596-912c8f91a37b kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4.18a6e5312b261778 namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: a02e39e7-c7df-4eff-b719-99e14c4be8e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-df533-predictor-84d8c6bb56 namespace: kserve-ci-e2e-test resourceVersion: "26772" uid: c09e2284-9998-4a4c-96fc-e7bc25b5ad2c kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: 'Created pod: error-404-isvc-df533-predictor-84d8c6bb56-gqkw4' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor-84d8c6bb56.18a6e51e7b4b76c6 namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: f35b0c3c-16f0-4d31-b881-5c7aab0c02c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-df533-predictor namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 78022598-0987-487d-a591-72e328a62382 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Scaled up replica set error-404-isvc-df533-predictor-84d8c6bb56 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533-predictor.18a6e51e787bf36f namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: db272295-a3d6-42b8-8d08-0643554f4930 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:03:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df533-predictor namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: c5f507c0-0499-4709-ab27-66595b340820 kind: Event lastTimestamp: "2026-04-16T17:04:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:57Z" name: error-404-isvc-df533-predictor.18a6e521f6d2e086 namespace: kserve-ci-e2e-test resourceVersion: "27597" uid: eb69f342-fa8a-42f1-9d0f-daa9c719f56a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:03:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df533-predictor namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: c5f507c0-0499-4709-ab27-66595b340820 kind: Event lastTimestamp: "2026-04-16T17:04:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:57Z" name: error-404-isvc-df533-predictor.18a6e521f73dcd37 namespace: kserve-ci-e2e-test resourceVersion: "27598" uid: 95c40597-f9d3-4eb0-9322-3c2205a12669 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 3b08e51e-4cb8-498b-b96a-16ebc2992f36 kind: Event lastTimestamp: "2026-04-16T17:04:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:33Z" name: error-404-isvc-df533.18a6e51e7821cf19 namespace: kserve-ci-e2e-test resourceVersion: "27368" uid: 1e19f7b9-13f8-4346-aa5f-1fbd1da2a4e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 3b08e51e-4cb8-498b-b96a-16ebc2992f36 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-df533": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df533": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533.18a6e51e78eb4930 namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: 87c10791-8450-4fbf-bbc6-493eeae324e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 3b08e51e-4cb8-498b-b96a-16ebc2992f36 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df533": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:42Z" name: error-404-isvc-df533.18a6e51e78eb9bc2 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 9ed614f6-cc4e-4f68-8df1-5035c7bfef8c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: 3b08e51e-4cb8-498b-b96a-16ebc2992f36 kind: Event lastTimestamp: "2026-04-16T17:04:33Z" message: InferenceService [error-404-isvc-df533] is Ready metadata: creationTimestamp: "2026-04-16T17:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:33Z" name: error-404-isvc-df533.18a6e52a78b2bffc namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: c194858e-dd77-4c8d-b9f4-b48ac77bbd57 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:41:56.407657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84f646df87-ws2ml to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:41:56Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3ee7880c413 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: 0200a3c8-3eec-42e8-98a1-4b1f157c01df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:41:56Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3ee917f94f4 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: fd922992-56a3-4bda-b325-e94f0743fa31 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:9cb363f3a01ee53202c49d182f025a6cf697ac6f3bd0a3c8aef7b04a0a53d808" metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:41:56Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3ee94800b5b namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: f85059bc-329b-4dd3-a20b-24d4eed00314 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:9cb363f3a01ee53202c49d182f025a6cf697ac6f3bd0a3c8aef7b04a0a53d808" in 14.272s (14.272s including waiting). Image size: 300960178 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f1e7381511 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 12f104eb-0339-44cd-b7ce-8ed7cea946d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f1eb733032 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: e85413ac-161d-40bd-83e3-93c6a64e5cb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f1ec17a186 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 1c4052b6-cc0c-406e-9a74-d99edf1ff0c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:14Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T16:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:14Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f2c8f60116 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 07cc0e52-f1dc-4d0c-9dee-628ecf76f0c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" in 8.646s (8.646s including waiting). Image size: 1560732082 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:23Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f4cc57b733 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 23c925c1-d217-4bfb-83ff-8001f4ffb2e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:23Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f4d133ec18 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: affadc3e-342a-47c2-9766-5b2f92b32e2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:42:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:23Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f4d1e464e1 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: e5b94da2-fd91-46da-bea3-d62c340d8dbe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:43:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:24Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e3f4e3db7387 namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 4cd940ce-7074-457e-b2d6-931dbc50266a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 079ad2c9-be6c-443d-bb79-d883f22b2614 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml.18a6e40ccbc856e7 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: bded5f28-b576-414c-8d04-cd598cf87750 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84f646df87 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: ffe33617-1f13-470e-a802-7f89e421678c kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84f646df87-ws2ml' metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: isvc-sklearn-graph-1-predictor-84f646df87.18a6e3ee776e071f namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 187c7c79-fb25-4a29-94ef-59ffbd172975 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 04d92eb4-b7f3-4ffc-9440-99e233d68c83 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84f646df87 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: isvc-sklearn-graph-1-predictor.18a6e3ee710f6113 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 39143edc-0f37-4a9b-9a19-c83bb9ab81c4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: f0210594-a909-416c-bd46-46ed7cc8bf45 kind: Event lastTimestamp: "2026-04-16T16:42:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:26Z" name: isvc-sklearn-graph-1-predictor.18a6e3f1f770de2c namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: bbd7f37e-35b9-43ba-97db-3d7ec9736f8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: f0210594-a909-416c-bd46-46ed7cc8bf45 kind: Event lastTimestamp: "2026-04-16T16:42:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:26Z" name: isvc-sklearn-graph-1-predictor.18a6e3f1f7e0e8a9 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: f9862f96-89fc-4a54-91ce-4859fb069204 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: f0210594-a909-416c-bd46-46ed7cc8bf45 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:56Z" name: isvc-sklearn-graph-1-predictor.18a6e3f8f67a6dfa namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 23e188ef-1e83-4a9f-8b78-23caa3bfd34d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: f0210594-a909-416c-bd46-46ed7cc8bf45 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:56Z" name: isvc-sklearn-graph-1-predictor.18a6e3f8f6dbd300 namespace: kserve-ci-e2e-test resourceVersion: "16996" uid: 3ec85f56-dc60-4391-8147-93c1e752ae6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: 96d65c92-b0b4-4b60-8b81-1a3cab50cac2 kind: Event lastTimestamp: "2026-04-16T16:43:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:35Z" name: isvc-sklearn-graph-1.18a6e3ee7802d383 namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: e26a5e65-f42e-43ab-aae3-bab53c23e890 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 96d65c92-b0b4-4b60-8b81-1a3cab50cac2 kind: Event lastTimestamp: "2026-04-16T16:43:35Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-16T16:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:35Z" name: isvc-sklearn-graph-1.18a6e40574657bb8 namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: dc8cf8bc-cd0f-40cd-aaa1-3854aabe47f6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:41:57.519039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-555687cc47-2p67c to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:41:57Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3eebabf2599 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: fce33ee1-7052-4dac-bd66-14a361c1a569 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:41:58Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:41:58Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3eedb6382d8 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: a3e28f34-b616-4ec1-900b-9e24f0f1b072 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:41:58Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:9cb363f3a01ee53202c49d182f025a6cf697ac6f3bd0a3c8aef7b04a0a53d808" metadata: creationTimestamp: "2026-04-16T16:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:41:58Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3eedd74b91d namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: bf313fc7-ca25-4976-a3a5-0b7cd64c67c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:9cb363f3a01ee53202c49d182f025a6cf697ac6f3bd0a3c8aef7b04a0a53d808" in 13.049s (13.049s including waiting). Image size: 300960178 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f1e7471546 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: f05dd6df-aafe-4b2d-9b96-63db01701eef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f1eb80e582 namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: 85c30411-30b5-4ea2-8dbb-702238c9cb48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f1ebf95998 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: df16d0c8-557c-47af-aaad-63e737f36066 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:15Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T16:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:15Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f304eeba0e namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: dc4d194f-2e89-4039-b339-2d95f1c5048d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1324" in 7.658s (7.658s including waiting). Image size: 1560732082 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:23Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f4cd6bca14 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: a66bdeb7-094a-4264-874d-999bbc8914c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:23Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f4d192ddc8 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 56cc07f7-5f97-4a71-854c-96f22785d2f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:42:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:23Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f4d21327c3 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 08dba5d2-0267-4263-9b2d-3fdacc41cdc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:43:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:24Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e3f4e3f3c4de namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: f01602d8-6848-4965-a80d-b2e32d345d02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 30afbb56-ef59-4826-802f-2efdf8c52792 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: isvc-sklearn-graph-2-predictor-555687cc47-2p67c.18a6e40cc8792fd9 namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: 576713da-f08d-4a5b-ae51-4dcac068b06c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-555687cc47 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 1bd7b77f-523f-47fb-89be-b88392e689ad kind: Event lastTimestamp: "2026-04-16T16:41:57Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-555687cc47-2p67c' metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:57Z" name: isvc-sklearn-graph-2-predictor-555687cc47.18a6e3eeb9062268 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 85a0b809-c9ee-40a5-a36d-4b4263fbd0a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 3f85cc07-2904-4360-953e-41e9681ab8db kind: Event lastTimestamp: "2026-04-16T16:41:57Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-555687cc47 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:57Z" name: isvc-sklearn-graph-2-predictor.18a6e3eea3805908 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: d9f06362-e689-493d-bc3d-f1f28b1398e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:42:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 9cd41e89-49b5-4ef6-abea-57544337d251 kind: Event lastTimestamp: "2026-04-16T16:42:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:57Z" name: isvc-sklearn-graph-2-predictor.18a6e3f22238a246 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: c4c51d6a-2191-4b23-891d-083ebe5363b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:42:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 9cd41e89-49b5-4ef6-abea-57544337d251 kind: Event lastTimestamp: "2026-04-16T16:42:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:57Z" name: isvc-sklearn-graph-2-predictor.18a6e3f222aa427a namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: c1043c43-63b3-401c-b4a2-ce0e0f68f0f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:43:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 9cd41e89-49b5-4ef6-abea-57544337d251 kind: Event lastTimestamp: "2026-04-16T16:43:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:42Z" name: isvc-sklearn-graph-2-predictor.18a6e4001efa9449 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 250c5edf-c4f4-4a45-8d3d-bb5e97242bf8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:43:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 9cd41e89-49b5-4ef6-abea-57544337d251 kind: Event lastTimestamp: "2026-04-16T16:43:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:42Z" name: isvc-sklearn-graph-2-predictor.18a6e4001f863daa namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: ced96ed9-700a-4ab5-83b0-76bca0f8d3a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 69151930-411a-4e47-b061-340bd2bbaa47 kind: Event lastTimestamp: "2026-04-16T16:43:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:35Z" name: isvc-sklearn-graph-2.18a6e3eea39e197f namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: cb5fc4fe-29c7-46ad-9092-970a862c479e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: 69151930-411a-4e47-b061-340bd2bbaa47 kind: Event lastTimestamp: "2026-04-16T16:43:35Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-16T16:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:35Z" name: isvc-sklearn-graph-2.18a6e40571478030 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: e7cc5635-d534-4e66-a87d-b322ced7115e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:41:56.892651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-b6mw4 to ip-10-0-128-64.ec2.internal metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:41:56Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3ee956928d0 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 2c69678e-84bd-4a82-90a4-b2d5e13e45e3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:41:57Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:41:57Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3eeae4048c9 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: 3bb38441-2bcc-4515-a8d0-a18b8a575724 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:41:57Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:9cb363f3a01ee53202c49d182f025a6cf697ac6f3bd0a3c8aef7b04a0a53d808" metadata: creationTimestamp: "2026-04-16T16:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:41:57Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3eeb184e8e2 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 5cba4575-d54b-4a82-b1f0-9d451b2b7ca0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:9cb363f3a01ee53202c49d182f025a6cf697ac6f3bd0a3c8aef7b04a0a53d808" in 13.786s (13.786s including waiting). Image size: 300960178 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f1e7476d37 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 7114f073-f7d3-4544-a2b6-4bb622108992 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f1eb9aa975 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: ec0bbe8f-253f-4df3-b8ea-15837f77a4a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:11Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f1ec268bae namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: a26f3e3a-322b-4221-b68b-253196153168 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:15Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T16:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:15Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f304df8eca namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: a2f97653-9c79-4d69-868c-9b98125c04c6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:34Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.73s (18.73s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:34Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f7614a175f namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 1f304d3f-b907-45c8-aeaf-9cca7aa44d57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:34Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f7664042f3 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 2da860c8-ad3d-4e04-9243-7d38a82dd7ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:42:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:34Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f766ce9787 namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: 74879cca-fc82-46ac-b0ad-5852b342d963 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T16:42:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:43:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:42:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:36Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e3f775edc5ec namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 4086301b-f75f-4add-9acc-5a6d56b0384a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: a8ae0449-aec8-4633-997a-7137047bd4ae kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: isvc-xgboost-graph-predictor-784cb989b8-b6mw4.18a6e40cc4f7b03d namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: 70bd9b7e-1c74-484a-aeb8-3488227d8218 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-64.ec2.internal source: component: kubelet host: ip-10-0-128-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: d590f67c-8151-4df9-98e5-c35f5ed77c13 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-b6mw4' metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a6e3ee9479c594 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 5bf5305b-ad07-477d-ba5e-40836256e360 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: e8d6aa63-7326-4225-aece-13ca57a9492e kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: isvc-xgboost-graph-predictor.18a6e3ee8b890bd6 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: b6433864-edb9-4bf4-894e-5766224f1ec3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 0ffc9273-d66f-4100-87d0-2768e5e58c01 kind: Event lastTimestamp: "2026-04-16T16:42:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:56Z" name: isvc-xgboost-graph-predictor.18a6e3f20a779567 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: de72aebe-8476-4ac1-8b81-82f535bc1111 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 0ffc9273-d66f-4100-87d0-2768e5e58c01 kind: Event lastTimestamp: "2026-04-16T16:42:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:56Z" name: isvc-xgboost-graph-predictor.18a6e3f20ae7616e namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: f59e323f-ad0f-4280-879e-19ff27ece2e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:43:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 0ffc9273-d66f-4100-87d0-2768e5e58c01 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:43:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:56Z" name: isvc-xgboost-graph-predictor.18a6e40005f31bba namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 925cc0fc-d799-42c1-991b-dac082cf3246 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:43:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 0ffc9273-d66f-4100-87d0-2768e5e58c01 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:43:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:56Z" name: isvc-xgboost-graph-predictor.18a6e4000676aa0d namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: dbfd4e8a-97de-4391-8455-cd23ecc24874 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: b82af773-ee46-4c9d-81ee-e5bc5e127ab1 kind: Event lastTimestamp: "2026-04-16T16:43:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:46Z" name: isvc-xgboost-graph.18a6e3ee8bcd4300 namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: ee261b09-d803-4131-8e62-dc53d1ebd40d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: b82af773-ee46-4c9d-81ee-e5bc5e127ab1 kind: Event lastTimestamp: "2026-04-16T16:43:46Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-16T16:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:46Z" name: isvc-xgboost-graph.18a6e40802df7ad1 namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: abea4377-01d2-49b5-b129-39d8c0c2709a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:43:56.405908Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16978" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-cb97498c4-rrszv to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:43:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:43:56Z" name: model-chainer-cb97498c4-rrszv.18a6e40a68f4cdd9 namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: abf151fc-5177-4430-9604-171508668485 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:56Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T16:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:56Z" name: model-chainer-cb97498c4-rrszv.18a6e40a721a4df0 namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: e621ec11-1013-4876-b68b-f8e43c4ca283 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:57Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16986" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:43:57Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:43:57Z" name: model-chainer-cb97498c4-rrszv.18a6e40aa615fe5a namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: b1de15af-3ec3-4196-8e44-940837c5527c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:43:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:57Z" name: model-chainer-cb97498c4-rrszv.18a6e40aa752d769 namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: 33990400-8495-41a8-8772-804120a73f9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:43:57Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-16T16:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:57Z" name: model-chainer-cb97498c4-rrszv.18a6e40aaa95f796 namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 3a0b5e11-5fe5-4b86-a799-8377108985b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:43:57Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-16T16:43:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:57Z" name: model-chainer-cb97498c4-rrszv.18a6e40aab0b7e0d namespace: kserve-ci-e2e-test resourceVersion: "17010" uid: 05ad85ca-8aa3-4c17-90d7-1804d76c410c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: model-chainer-cb97498c4-rrszv.18a6e40cc0954fc0 namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: c2806b3a-7bbe-4bd3-bc98-7176dcd257ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:44:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-cb97498c4-rrszv namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 3e269716-9684-4956-b65c-7817600e2419 kind: Event lastTimestamp: "2026-04-16T16:44:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:44:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:33Z" name: model-chainer-cb97498c4-rrszv.18a6e40d4031ff04 namespace: kserve-ci-e2e-test resourceVersion: "17472" uid: 8fe47c7a-76a2-4f5a-a9b5-6dc8487da3ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-cb97498c4 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: a43a4a5a-2eec-4004-87e0-9665fec29dc7 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: 'Created pod: model-chainer-cb97498c4-rrszv' metadata: creationTimestamp: "2026-04-16T16:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:56Z" name: model-chainer-cb97498c4.18a6e40a6865e81c namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 086a595f-1d4b-4cac-b650-cdc6b740addd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 375c48c7-80c5-406e-a47d-92e01bc35402 kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: Scaled up replica set model-chainer-cb97498c4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:56Z" name: model-chainer.18a6e40a66181ead namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: 773be2f9-2778-42fc-94eb-3621213d416f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 5058a790-9791-41c5-8b17-78099379b6bb kind: Event lastTimestamp: "2026-04-16T16:43:56Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-16T16:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:56Z" name: model-chainer.18a6e40a6bb4c7c9 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 55449b67-e743-45e2-8d00-f3659df90b65 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:54:41.696852Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22327" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-04ea1-5fbb7b7bc6-nwwbf to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:54:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:54:41Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a0a74b22cd namespace: kserve-ci-e2e-test resourceVersion: "22332" uid: fa5098f6-14d2-436a-a873-5917948baf92 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22336" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: "2026-04-16T16:54:42Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:54:42Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a0c0ec4fee namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: b91cef86-5a5e-4c6e-8ae8-7124066086ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-04ea1} kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: "2026-04-16T16:54:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:42Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a0c238e6d3 namespace: kserve-ci-e2e-test resourceVersion: "22349" uid: 1b5b204d-9287-4636-ad05-636633ac6a19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-04ea1} kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: "2026-04-16T16:54:42Z" message: 'Created container: sequence-graph-04ea1' metadata: creationTimestamp: "2026-04-16T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:42Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a0c5398bd1 namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: b78559c3-63e0-49f5-ab8c-6375739e79bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-04ea1} kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: "2026-04-16T16:54:42Z" message: Started container sequence-graph-04ea1 metadata: creationTimestamp: "2026-04-16T16:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:42Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a0c5af19b6 namespace: kserve-ci-e2e-test resourceVersion: "22353" uid: cdbf2548-26b5-43d6-8955-65e0830ae2f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-04ea1} kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: "2026-04-16T16:54:51Z" message: Stopping container sequence-graph-04ea1 metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:51Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a3002fe2be namespace: kserve-ci-e2e-test resourceVersion: "22466" uid: aca4507e-7d1a-4f0b-abe9-39b1f75d4319 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-04ea1} kind: Pod name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: 37d34243-13cb-4b01-ae05-97a2c9e61d31 kind: Event lastTimestamp: "2026-04-16T16:55:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:55:18Z" name: sequence-graph-04ea1-5fbb7b7bc6-nwwbf.18a6e4a378a9a9f3 namespace: kserve-ci-e2e-test resourceVersion: "22814" uid: ce82911d-bdf9-4dd3-8eaa-18cd90fdbd88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-04ea1-5fbb7b7bc6 namespace: kserve-ci-e2e-test resourceVersion: "22324" uid: ebeb16d6-7a6d-43fc-8c3e-ccb07b6d9dc0 kind: Event lastTimestamp: "2026-04-16T16:54:41Z" message: 'Created pod: sequence-graph-04ea1-5fbb7b7bc6-nwwbf' metadata: creationTimestamp: "2026-04-16T16:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:41Z" name: sequence-graph-04ea1-5fbb7b7bc6.18a6e4a0a6c38130 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 82a8ade1-da09-4acb-85dd-7c4a9f8e4ef3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "22322" uid: 86aeaad3-6b01-4bdf-9439-f3d9797c0af8 kind: Event lastTimestamp: "2026-04-16T16:54:41Z" message: Scaled up replica set sequence-graph-04ea1-5fbb7b7bc6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:41Z" name: sequence-graph-04ea1.18a6e4a0a5f94cd0 namespace: kserve-ci-e2e-test resourceVersion: "22325" uid: f3255b4c-6b58-486f-9f7f-92ccd93c81ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: cdd6d0dd-14da-433d-8d67-00266625d3e7 kind: Event lastTimestamp: "2026-04-16T16:54:41Z" message: InferenceGraph [sequence-graph-04ea1] is Ready metadata: creationTimestamp: "2026-04-16T16:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:41Z" name: sequence-graph-04ea1.18a6e4a0aa21ca8c namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 95f8e30e-4517-4cbf-877f-4ca807a055ef reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:45:16.665296Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-39f4a-89d49ccf6-5mbjd to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:45:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:45:16Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e41d18c9c874 namespace: kserve-ci-e2e-test resourceVersion: "17880" uid: 3922d47a-382a-4b43-ae6f-064c6e2bb995 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:45:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-39f4a-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:45:16Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e41d2aa4d601 namespace: kserve-ci-e2e-test resourceVersion: "17894" uid: 9fb98885-da87-42b5-ab95-cf0a69def9b1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17884" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:45:17Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:45:17Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e41d56135b67 namespace: kserve-ci-e2e-test resourceVersion: "17903" uid: 1925f424-5ca0-49cf-809b-67bd11ac06a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39f4a} kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:45:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:45:17Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e41d57575710 namespace: kserve-ci-e2e-test resourceVersion: "17905" uid: 1ccd02d9-9c89-4cfc-ba19-6fc7f72a441a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39f4a} kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:45:17Z" message: 'Created container: sequence-graph-39f4a' metadata: creationTimestamp: "2026-04-16T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:45:17Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e41d5ac25999 namespace: kserve-ci-e2e-test resourceVersion: "17906" uid: e93c2705-3d1f-4c74-8552-8de9bc7bd7c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39f4a} kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:45:17Z" message: Started container sequence-graph-39f4a metadata: creationTimestamp: "2026-04-16T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:45:17Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e41d5b33e6dd namespace: kserve-ci-e2e-test resourceVersion: "17907" uid: 1053de88-c535-4fb6-885a-1772f759d8be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39f4a} kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Stopping container sequence-graph-39f4a metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e4905189262c namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 0e2a7cf9-6070-4137-86f9-e84e77610faa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-39f4a} kind: Pod name: sequence-graph-39f4a-89d49ccf6-5mbjd namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cc5b1b60-9892-47e3-b0b6-e0f3f8918f52 kind: Event lastTimestamp: "2026-04-16T16:53:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:58Z" name: sequence-graph-39f4a-89d49ccf6-5mbjd.18a6e490ddaefb67 namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 2c2150ce-e11b-4c79-9e60-e07f3b4db804 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-39f4a-89d49ccf6 namespace: kserve-ci-e2e-test resourceVersion: "17872" uid: 71ad9f11-4291-4903-83e6-8bcb18d1eb79 kind: Event lastTimestamp: "2026-04-16T16:45:16Z" message: 'Created pod: sequence-graph-39f4a-89d49ccf6-5mbjd' metadata: creationTimestamp: "2026-04-16T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:16Z" name: sequence-graph-39f4a-89d49ccf6.18a6e41d185dbe14 namespace: kserve-ci-e2e-test resourceVersion: "17876" uid: ed96ccb6-a692-4cec-a5bc-2af66f10de26 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17870" uid: bad49d9d-9a7d-4550-b072-e755c90eb546 kind: Event lastTimestamp: "2026-04-16T16:45:16Z" message: Scaled up replica set sequence-graph-39f4a-89d49ccf6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:16Z" name: sequence-graph-39f4a.18a6e41d179b74ac namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: 4d5d3e39-1b03-47c3-9b27-0c4e89293604 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: 08e08322-10cf-42df-b871-0a801ecd7625 kind: Event lastTimestamp: "2026-04-16T16:45:16Z" message: InferenceGraph [sequence-graph-39f4a] is Ready metadata: creationTimestamp: "2026-04-16T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:45:16Z" name: sequence-graph-39f4a.18a6e41d1d5f41b2 namespace: kserve-ci-e2e-test resourceVersion: "17889" uid: 9897a6f5-5364-4375-b720-11ebd46d0757 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17871" uid: cb3f657b-eb85-477e-840a-aabc706dfa97 kind: Event lastTimestamp: "2026-04-16T16:45:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:31Z" name: sequence-graph-39f4a.18a6e420962f459a namespace: kserve-ci-e2e-test resourceVersion: "18013" uid: 3b33c1b4-d641-46f4-8591-0c0740e32a7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17871" uid: cb3f657b-eb85-477e-840a-aabc706dfa97 kind: Event lastTimestamp: "2026-04-16T16:45:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:31Z" name: sequence-graph-39f4a.18a6e42096807fb0 namespace: kserve-ci-e2e-test resourceVersion: "18015" uid: f6ada7b1-7e5d-4fee-9a0e-a5d64e7cf0d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-39f4a namespace: kserve-ci-e2e-test resourceVersion: "18014" uid: cb3f657b-eb85-477e-840a-aabc706dfa97 kind: Event lastTimestamp: "2026-04-16T16:45:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:46Z" name: sequence-graph-39f4a.18a6e4241547db5a namespace: kserve-ci-e2e-test resourceVersion: "18123" uid: aa5d562c-8c0d-43ff-9682-a91c2f0370a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:45:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-39f4a namespace: kserve-ci-e2e-test resourceVersion: "18014" uid: cb3f657b-eb85-477e-840a-aabc706dfa97 kind: Event lastTimestamp: "2026-04-16T16:45:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:46Z" name: sequence-graph-39f4a.18a6e42415b653eb namespace: kserve-ci-e2e-test resourceVersion: "18125" uid: 6aafe874-4757-4554-b307-6547ea85e991 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:56:02.075037Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23215" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-64f8d-7d567c889c-8np26 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:56:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:56:02Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e4b35e34f5ad namespace: kserve-ci-e2e-test resourceVersion: "23220" uid: a8a6d028-7cba-42fe-a80a-b623a86af180 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23216" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T16:56:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-64f8d-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T16:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:56:02Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e4b369b2f14f namespace: kserve-ci-e2e-test resourceVersion: "23230" uid: de7ffee9-587b-4737-b1ad-7fa7f7964a50 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23224" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T16:56:03Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:56:03Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e4b39b60afa4 namespace: kserve-ci-e2e-test resourceVersion: "23241" uid: e0b46288-240d-4d55-8352-b0ad7370a525 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64f8d} kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23216" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T16:56:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:56:03Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e4b39ce7cda9 namespace: kserve-ci-e2e-test resourceVersion: "23243" uid: 9358fbe4-cbe5-4164-87e2-aa6e76280ad2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64f8d} kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23216" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T16:56:03Z" message: 'Created container: sequence-graph-64f8d' metadata: creationTimestamp: "2026-04-16T16:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:56:03Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e4b3a0a0fd3c namespace: kserve-ci-e2e-test resourceVersion: "23246" uid: 32125d78-babb-44d6-a84b-d7a384be917e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64f8d} kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23216" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T16:56:03Z" message: Started container sequence-graph-64f8d metadata: creationTimestamp: "2026-04-16T16:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:56:03Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e4b3a1143ce4 namespace: kserve-ci-e2e-test resourceVersion: "23247" uid: c11beac7-ffd6-471c-9e70-c02ba68454cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64f8d} kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23216" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: Stopping container sequence-graph-64f8d metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:16Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e5268d1ad4dd namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: f0da0025-ba4f-40ac-8a49-2743ff8430ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64f8d} kind: Pod name: sequence-graph-64f8d-7d567c889c-8np26 namespace: kserve-ci-e2e-test resourceVersion: "23216" uid: b8b9877e-9a55-4f2c-92c8-af577bba4326 kind: Event lastTimestamp: "2026-04-16T17:04:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:45Z" name: sequence-graph-64f8d-7d567c889c-8np26.18a6e52751a6e275 namespace: kserve-ci-e2e-test resourceVersion: "27461" uid: 5d887cd9-f5af-4845-aeea-8251ea543964 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-64f8d-7d567c889c namespace: kserve-ci-e2e-test resourceVersion: "23212" uid: 5a13c5a2-af35-4325-9384-4ace871e7ff7 kind: Event lastTimestamp: "2026-04-16T16:56:02Z" message: 'Created pod: sequence-graph-64f8d-7d567c889c-8np26' metadata: creationTimestamp: "2026-04-16T16:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:02Z" name: sequence-graph-64f8d-7d567c889c.18a6e4b35d8beed8 namespace: kserve-ci-e2e-test resourceVersion: "23217" uid: a52917c5-be5e-4359-b592-ebb3f5e17b23 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23210" uid: 6ec9047a-b9a6-4ff3-a07c-0b5f10ba0d6f kind: Event lastTimestamp: "2026-04-16T16:56:02Z" message: Scaled up replica set sequence-graph-64f8d-7d567c889c from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:02Z" name: sequence-graph-64f8d.18a6e4b35bbbbf0e namespace: kserve-ci-e2e-test resourceVersion: "23213" uid: 0fecc687-fa6d-45c4-9671-850f8fdfaec1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23227" uid: 5c02dfb1-29d1-400f-82ef-6802204dc769 kind: Event lastTimestamp: "2026-04-16T16:56:02Z" message: InferenceGraph [sequence-graph-64f8d] is Ready metadata: creationTimestamp: "2026-04-16T16:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:56:02Z" name: sequence-graph-64f8d.18a6e4b366ae867e namespace: kserve-ci-e2e-test resourceVersion: "23229" uid: 4875b041-3b9a-4a87-9127-407064f1b18d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23211" uid: ad94dab4-d2c0-4483-a76d-167a443895a4 kind: Event lastTimestamp: "2026-04-16T16:56:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:17Z" name: sequence-graph-64f8d.18a6e4b6db7bf21d namespace: kserve-ci-e2e-test resourceVersion: "23355" uid: 0332fb04-e719-4ac8-a28b-975befd7c977 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23211" uid: ad94dab4-d2c0-4483-a76d-167a443895a4 kind: Event lastTimestamp: "2026-04-16T16:56:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:17Z" name: sequence-graph-64f8d.18a6e4b6dc376259 namespace: kserve-ci-e2e-test resourceVersion: "23357" uid: 20ddddb8-9e46-4de0-82fb-00d084f1e75d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23356" uid: ad94dab4-d2c0-4483-a76d-167a443895a4 kind: Event lastTimestamp: "2026-04-16T16:56:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:32Z" name: sequence-graph-64f8d.18a6e4ba5b08f97a namespace: kserve-ci-e2e-test resourceVersion: "23465" uid: 34d839ca-10b6-47a8-a71d-7ea1a1fef1e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:56:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23356" uid: ad94dab4-d2c0-4483-a76d-167a443895a4 kind: Event lastTimestamp: "2026-04-16T16:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:32Z" name: sequence-graph-64f8d.18a6e4ba5b74e64f namespace: kserve-ci-e2e-test resourceVersion: "23466" uid: 070249b7-5bb4-4c9e-af67-259bd596b6ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:06:12.410665Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28401" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-45e11-8475c5d8c4-ngt97 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T17:06:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:06:12Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e541790b86ea namespace: kserve-ci-e2e-test resourceVersion: "28406" uid: 6322a0d3-c042-479a-9518-a91979a3f484 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28410" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: "2026-04-16T17:06:13Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:06:13Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e5419e8f46c5 namespace: kserve-ci-e2e-test resourceVersion: "28421" uid: 08f7d09e-720b-4e87-b711-18b41e887288 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-45e11} kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28403" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: "2026-04-16T17:06:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T17:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:13Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e5419fdf5fbf namespace: kserve-ci-e2e-test resourceVersion: "28423" uid: 1b185b27-3199-4b7a-9382-a570ea6b00b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-45e11} kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28403" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: "2026-04-16T17:06:13Z" message: 'Created container: splitter-graph-45e11' metadata: creationTimestamp: "2026-04-16T17:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:13Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e541a2ae819b namespace: kserve-ci-e2e-test resourceVersion: "28424" uid: 5e8cd07b-de23-42c9-9c04-60db542df168 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-45e11} kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28403" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: "2026-04-16T17:06:13Z" message: Started container splitter-graph-45e11 metadata: creationTimestamp: "2026-04-16T17:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:06:13Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e541a31f58f9 namespace: kserve-ci-e2e-test resourceVersion: "28425" uid: 419a2985-12c5-45fe-bec0-12e0b24fa86f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-45e11} kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28403" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: "2026-04-16T17:14:27Z" message: Stopping container splitter-graph-45e11 metadata: creationTimestamp: "2026-04-16T17:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:14:27Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e5b4a107511a namespace: kserve-ci-e2e-test resourceVersion: "31808" uid: c33fd4b6-0a28-42c2-a944-7acd4605fae7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:14:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-45e11} kind: Pod name: splitter-graph-45e11-8475c5d8c4-ngt97 namespace: kserve-ci-e2e-test resourceVersion: "28403" uid: 7ca2f9d4-0e4e-4a07-814a-13eb9ca82d80 kind: Event lastTimestamp: "2026-04-16T17:14:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:14:54Z" name: splitter-graph-45e11-8475c5d8c4-ngt97.18a6e5b52055d8c0 namespace: kserve-ci-e2e-test resourceVersion: "32036" uid: fd8672c1-e235-4c7c-a38f-f84a20ad70fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-45e11-8475c5d8c4 namespace: kserve-ci-e2e-test resourceVersion: "28398" uid: 52419935-2930-482c-b58c-580de5dd1c78 kind: Event lastTimestamp: "2026-04-16T17:06:12Z" message: 'Created pod: splitter-graph-45e11-8475c5d8c4-ngt97' metadata: creationTimestamp: "2026-04-16T17:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:12Z" name: splitter-graph-45e11-8475c5d8c4.18a6e541788e3fe0 namespace: kserve-ci-e2e-test resourceVersion: "28402" uid: e9de25ff-ac81-40cb-b91b-7591e446f4a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28396" uid: 63e66205-1e06-47dd-b4c5-27d10de1bbd3 kind: Event lastTimestamp: "2026-04-16T17:06:12Z" message: Scaled up replica set splitter-graph-45e11-8475c5d8c4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:12Z" name: splitter-graph-45e11.18a6e54177cd21da namespace: kserve-ci-e2e-test resourceVersion: "28399" uid: c70a69da-20f6-4d0d-add6-ea98826c6bb6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28412" uid: b25f6e04-0566-4ceb-bcce-a258dd78c4c6 kind: Event lastTimestamp: "2026-04-16T17:06:12Z" message: InferenceGraph [splitter-graph-45e11] is Ready metadata: creationTimestamp: "2026-04-16T17:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:06:12Z" name: splitter-graph-45e11.18a6e5417ba2508d namespace: kserve-ci-e2e-test resourceVersion: "28414" uid: 2b94b819-168d-4bfb-af8c-1b0dbf4e3f89 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28397" uid: 114361f8-5208-4a58-86b6-163955ab21e9 kind: Event lastTimestamp: "2026-04-16T17:06:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:27Z" name: splitter-graph-45e11.18a6e544f650632c namespace: kserve-ci-e2e-test resourceVersion: "28538" uid: 8b59203f-b86f-48ec-9b14-60db9e48cdfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28397" uid: 114361f8-5208-4a58-86b6-163955ab21e9 kind: Event lastTimestamp: "2026-04-16T17:06:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:27Z" name: splitter-graph-45e11.18a6e544f6bbde04 namespace: kserve-ci-e2e-test resourceVersion: "28540" uid: 7826cc58-5cef-4eee-871b-240b7d1b120b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28539" uid: 114361f8-5208-4a58-86b6-163955ab21e9 kind: Event lastTimestamp: "2026-04-16T17:06:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:42Z" name: splitter-graph-45e11.18a6e5487565e177 namespace: kserve-ci-e2e-test resourceVersion: "28640" uid: c14c5c35-d9c8-4b0c-97d7-dfd7c747e206 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:06:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28539" uid: 114361f8-5208-4a58-86b6-163955ab21e9 kind: Event lastTimestamp: "2026-04-16T17:06:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:42Z" name: splitter-graph-45e11.18a6e54875afad15 namespace: kserve-ci-e2e-test resourceVersion: "28642" uid: f7be9e8e-7d3d-479c-aaee-a9c38bc69ce1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:04:52.145854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-df533-65fbb7d4f-xpzqr to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T17:04:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:04:52Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e52ec8e3b7ab namespace: kserve-ci-e2e-test resourceVersion: "27542" uid: 69b22e97-ab5c-4590-8d5f-4168ad6edbd1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:04:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-df533-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T17:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:52Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e52ed444f113 namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: 3af081a0-d3fc-4aaa-9d71-72db2d211276 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27545" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:04:53Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:04:53Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e52f0627284d namespace: kserve-ci-e2e-test resourceVersion: "27564" uid: 59990096-62c3-4a03-b507-040fa7d5f32a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-df533} kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:04:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T17:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:53Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e52f0780a0a3 namespace: kserve-ci-e2e-test resourceVersion: "27566" uid: f5d4dc48-2e6b-4082-ac49-9b2bf68c8c6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-df533} kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:04:53Z" message: 'Created container: splitter-graph-df533' metadata: creationTimestamp: "2026-04-16T17:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:53Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e52f0aeed145 namespace: kserve-ci-e2e-test resourceVersion: "27567" uid: 7392fbba-3bac-49d1-a56a-887acb9690bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-df533} kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:04:53Z" message: Started container splitter-graph-df533 metadata: creationTimestamp: "2026-04-16T17:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:53Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e52f0b633bd6 namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 8efd8ce0-c626-47a1-9723-a5418b2b2292 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-df533} kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Stopping container splitter-graph-df533 metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e5311ff6071f namespace: kserve-ci-e2e-test resourceVersion: "27659" uid: 86c2feea-13f2-4fae-b4ec-9d4379f00764 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-df533} kind: Pod name: splitter-graph-df533-65fbb7d4f-xpzqr namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: d62cd1a9-7469-4121-aee9-c23889606c61 kind: Event lastTimestamp: "2026-04-16T17:05:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:29Z" name: splitter-graph-df533-65fbb7d4f-xpzqr.18a6e531bf0ff90d namespace: kserve-ci-e2e-test resourceVersion: "28038" uid: 4934840e-35fd-4dc1-bd04-323559960304 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-df533-65fbb7d4f namespace: kserve-ci-e2e-test resourceVersion: "27533" uid: eaf2ddab-d33b-4b58-9d48-f4fd11d6ba4c kind: Event lastTimestamp: "2026-04-16T17:04:52Z" message: 'Created pod: splitter-graph-df533-65fbb7d4f-xpzqr' metadata: creationTimestamp: "2026-04-16T17:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:52Z" name: splitter-graph-df533-65fbb7d4f.18a6e52ec81790ba namespace: kserve-ci-e2e-test resourceVersion: "27538" uid: a02c8bd7-afbb-424a-aeef-979f18d394aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-df533 namespace: kserve-ci-e2e-test resourceVersion: "27531" uid: 0d5ecf3c-1e6d-43f0-9206-d258e8789c9f kind: Event lastTimestamp: "2026-04-16T17:04:52Z" message: Scaled up replica set splitter-graph-df533-65fbb7d4f from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:52Z" name: splitter-graph-df533.18a6e52ec7180232 namespace: kserve-ci-e2e-test resourceVersion: "27534" uid: a2c74528-4d79-47c9-8d54-e9dfe50f7b1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-df533 namespace: kserve-ci-e2e-test resourceVersion: "27547" uid: 12ed6a87-7b9e-400b-9b3b-20e03de0fe5d kind: Event lastTimestamp: "2026-04-16T17:04:52Z" message: InferenceGraph [splitter-graph-df533] is Ready metadata: creationTimestamp: "2026-04-16T17:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:52Z" name: splitter-graph-df533.18a6e52ecdb3bc6e namespace: kserve-ci-e2e-test resourceVersion: "27549" uid: 34fbd50f-6aec-41ba-854e-e86a8f635501 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:53:31.659437Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e49058bcf966 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: 17ef202d-a0fd-41a3-9e3b-89c1ee104467 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e490613d9b09 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: bd68a2f6-1127-4b9b-ba9b-376207428d76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e49062a4c951 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: abd299d0-5d15-42b9-b085-50cd58f6e4ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e49069652b5e namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 3d333a7f-59c1-4d94-83de-f3838705ca9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e49069dc01c8 namespace: kserve-ci-e2e-test resourceVersion: "21543" uid: f2ad6fc5-b688-4525-ae9e-bfcc8086975f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: "2026-04-16T16:54:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:53Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e4908d660ad6 namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: 4b32f2f5-764b-4107-99ff-c536d95754db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 9c8e9f31-7fa7-4a4b-a3b5-01a3fe1d19b9 kind: Event lastTimestamp: "2026-04-16T16:54:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:51Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k.18a6e4a306c4b093 namespace: kserve-ci-e2e-test resourceVersion: "22494" uid: 7090f3dc-caf1-4bdc-945f-f85fb7a86b61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-04ea1-predictor-6fcb9b4f57 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 654ac3e6-759c-437e-abb8-4090b25898c4 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'Created pod: success-200-isvc-04ea1-predictor-6fcb9b4f57-4p49k' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor-6fcb9b4f57.18a6e49057e2aa4b namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 021bee6f-b37b-4a59-b8eb-6b8826512515 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 8100d8ad-8daa-40d0-b39f-5df60227f90e kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Scaled up replica set success-200-isvc-04ea1-predictor-6fcb9b4f57 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1-predictor.18a6e4905538ebc3 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 4641caea-62d4-47e6-8231-339a6c870650 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 611f03be-68f5-45a3-a714-8001112f4c55 kind: Event lastTimestamp: "2026-04-16T16:53:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:46Z" name: success-200-isvc-04ea1-predictor.18a6e493d3f700a7 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: f23af049-a0db-455f-bc64-84dfb16eb520 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 611f03be-68f5-45a3-a714-8001112f4c55 kind: Event lastTimestamp: "2026-04-16T16:53:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:53:46Z" name: success-200-isvc-04ea1-predictor.18a6e493d447b2ea namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: f996c487-a5ee-472b-9255-d3179ed6bcb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:54:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 611f03be-68f5-45a3-a714-8001112f4c55 kind: Event lastTimestamp: "2026-04-16T16:54:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:46Z" name: success-200-isvc-04ea1-predictor.18a6e497531a85a7 namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: 52a2464a-498f-4818-ac65-0b7c2fecabdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T16:54:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04ea1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: 611f03be-68f5-45a3-a714-8001112f4c55 kind: Event lastTimestamp: "2026-04-16T16:54:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:46Z" name: success-200-isvc-04ea1-predictor.18a6e4975391e7eb namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: 1c899b25-fbf6-4514-b1e3-58a94fc89460 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: b6e01586-d328-4edd-a028-cc7de7c10ad5 kind: Event lastTimestamp: "2026-04-16T16:54:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:23Z" name: success-200-isvc-04ea1.18a6e4905532f56c namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 82c418fd-a42c-464a-88ec-d29968d6209e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: b6e01586-d328-4edd-a028-cc7de7c10ad5 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-04ea1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-04ea1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1.18a6e49055cf9e6f namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: 29b4194c-1a75-4e76-8933-ad363da2e65d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: b6e01586-d328-4edd-a028-cc7de7c10ad5 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-04ea1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-04ea1.18a6e49055cfff56 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: aef1c3f6-6869-416f-97ce-ad4e6e5e6d9d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04ea1 namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: b6e01586-d328-4edd-a028-cc7de7c10ad5 kind: Event lastTimestamp: "2026-04-16T16:54:23Z" message: InferenceService [success-200-isvc-04ea1] is Ready metadata: creationTimestamp: "2026-04-16T16:54:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:23Z" name: success-200-isvc-04ea1.18a6e49c71d67998 namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: 720b29de-41e6-4028-a061-4d2327703488 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:44:06.617050Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-39f4a-predictor-6b65487685-plh84 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e40cc9966497 namespace: kserve-ci-e2e-test resourceVersion: "17172" uid: 55bdaf27-77ae-45eb-8aa1-0eff2b147229 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e40cd227f548 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: a31424b1-11fa-4bfe-ab49-968117b317ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e40cd37971fb namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 84f9de9f-5a94-454c-9652-9282da534ce9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e40cd6f5674e namespace: kserve-ci-e2e-test resourceVersion: "17218" uid: ddec39dc-d852-4a07-b9f8-0f7a61078a08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e40cd76a05d7 namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: ca09a05f-6ce8-45b5-adfd-e5bab8401053 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:44:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: "2026-04-16T16:44:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:44:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:48Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e40d069d0e0f namespace: kserve-ci-e2e-test resourceVersion: "17629" uid: 31980123-a5ab-4f5c-b566-72f733f512a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-39f4a-predictor-6b65487685-plh84 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: aca8fe33-d187-45b7-b520-fcde24ee2234 kind: Event lastTimestamp: "2026-04-16T16:53:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:53:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:31Z" name: success-200-isvc-39f4a-predictor-6b65487685-plh84.18a6e4905b0da496 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: a54a2e84-0126-4616-ad10-f541473282ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-39f4a-predictor-6b65487685 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 74bc0051-d5db-4b25-a2bf-2b3226f77818 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: 'Created pod: success-200-isvc-39f4a-predictor-6b65487685-plh84' metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor-6b65487685.18a6e40cc90b7f43 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: fd66687e-75ac-4e4e-82aa-e9b837213670 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 6d6786ef-fb8a-4791-990c-091a13058f45 kind: Event lastTimestamp: "2026-04-16T16:44:06Z" message: Scaled up replica set success-200-isvc-39f4a-predictor-6b65487685 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:06Z" name: success-200-isvc-39f4a-predictor.18a6e40cc7486162 namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: d4b78411-7565-4083-9203-5e9bdebec8ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 8dfc8f59-ccac-4740-a21c-b049bdc9a6a6 kind: Event lastTimestamp: "2026-04-16T16:44:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:21Z" name: success-200-isvc-39f4a-predictor.18a6e41046130cc2 namespace: kserve-ci-e2e-test resourceVersion: "17364" uid: a9ab90d0-94cb-4744-a3be-a38f6a0d2d05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 8dfc8f59-ccac-4740-a21c-b049bdc9a6a6 kind: Event lastTimestamp: "2026-04-16T16:44:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:21Z" name: success-200-isvc-39f4a-predictor.18a6e4104694305b namespace: kserve-ci-e2e-test resourceVersion: "17365" uid: 5848bea7-76e5-46ed-bedf-aef74ec14eab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: 8dfc8f59-ccac-4740-a21c-b049bdc9a6a6 kind: Event lastTimestamp: "2026-04-16T16:45:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:36Z" name: success-200-isvc-39f4a-predictor.18a6e413c54f37d9 namespace: kserve-ci-e2e-test resourceVersion: "18047" uid: 9231c3e6-104c-401d-b474-208e7365c1fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:44:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-39f4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: 8dfc8f59-ccac-4740-a21c-b049bdc9a6a6 kind: Event lastTimestamp: "2026-04-16T16:45:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:36Z" name: success-200-isvc-39f4a-predictor.18a6e413c5c50e3e namespace: kserve-ci-e2e-test resourceVersion: "18048" uid: 8c24bf68-cb1f-43d9-9950-58e05596ab37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:44:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: 8e5ce877-a502-45d3-9a8c-c8c92162e299 kind: Event lastTimestamp: "2026-04-16T16:44:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:58Z" name: success-200-isvc-39f4a.18a6e40cc6d3babb namespace: kserve-ci-e2e-test resourceVersion: "17730" uid: b59e4e26-11fa-4d27-8017-971d5c3e7d28 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-39f4a namespace: kserve-ci-e2e-test resourceVersion: "17727" uid: 8e5ce877-a502-45d3-9a8c-c8c92162e299 kind: Event lastTimestamp: "2026-04-16T16:44:58Z" message: InferenceService [success-200-isvc-39f4a] is Ready metadata: creationTimestamp: "2026-04-16T16:44:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:58Z" name: success-200-isvc-39f4a.18a6e418eb369abf namespace: kserve-ci-e2e-test resourceVersion: "17728" uid: 237b867b-43f8-47ee-8412-b4d831e90fa0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:05:02.323182Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27688" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-45e11-predictor-686d55cc79-j4qw4 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e53127816a3f namespace: kserve-ci-e2e-test resourceVersion: "27696" uid: 3b6dad78-b15d-4bcd-8161-ddbc7c4d36a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27701" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e5312f9d145f namespace: kserve-ci-e2e-test resourceVersion: "27742" uid: 7acc9884-3ad4-414c-ab86-be06d785a58a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e53130c84407 namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: e291d083-3bb1-4162-a912-b13cf30c247d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e531343b3ce3 namespace: kserve-ci-e2e-test resourceVersion: "27749" uid: c584b1d0-a29c-4d8c-a4fd-68377e9c9043 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e53134aeae77 namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: 19dbcb65-426c-440d-9e45-22b3416301c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: "2026-04-16T17:05:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:43Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e531494c4269 namespace: kserve-ci-e2e-test resourceVersion: "28161" uid: 7b49ee46-01f6-4a76-848b-ea353ca93cf7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: be0b3717-18f2-4c58-aeb9-b15a1e676563 kind: Event lastTimestamp: "2026-04-16T17:14:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:14:27Z" name: success-200-isvc-45e11-predictor-686d55cc79-j4qw4.18a6e5b4a65f57d0 namespace: kserve-ci-e2e-test resourceVersion: "31824" uid: c275611a-4991-4462-a182-41d9f4a49a81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-45e11-predictor-686d55cc79 namespace: kserve-ci-e2e-test resourceVersion: "27675" uid: 65719ed4-165e-4fc1-a67d-066160a27055 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'Created pod: success-200-isvc-45e11-predictor-686d55cc79-j4qw4' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor-686d55cc79.18a6e53126cc677e namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: 0bc7948e-bec4-4ccb-b994-0e879b6c219c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27670" uid: b9e0de70-c423-4173-b4e8-e4f557c122ca kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Scaled up replica set success-200-isvc-45e11-predictor-686d55cc79 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11-predictor.18a6e5312475633b namespace: kserve-ci-e2e-test resourceVersion: "27678" uid: f881ab4f-c8b4-4f99-ae7d-b417b73ce9a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T17:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27674" uid: a02ab100-9464-494b-be82-c23d69aa21c6 kind: Event lastTimestamp: "2026-04-16T17:05:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:47Z" name: success-200-isvc-45e11-predictor.18a6e534a36f831a namespace: kserve-ci-e2e-test resourceVersion: "28189" uid: d54d34b1-9d04-4227-aad4-cb04a521ca39 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T17:05:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27674" uid: a02ab100-9464-494b-be82-c23d69aa21c6 kind: Event lastTimestamp: "2026-04-16T17:05:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:47Z" name: success-200-isvc-45e11-predictor.18a6e534a3e44c5a namespace: kserve-ci-e2e-test resourceVersion: "28190" uid: b3c1b4c3-a618-4736-8d43-06aa6b473bb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27913" uid: a02ab100-9464-494b-be82-c23d69aa21c6 kind: Event lastTimestamp: "2026-04-16T17:06:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:17Z" name: success-200-isvc-45e11-predictor.18a6e53f2038f0aa namespace: kserve-ci-e2e-test resourceVersion: "28457" uid: 685d8778-f9c1-4bc2-9c72-9041e2d5aee9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-45e11-predictor namespace: kserve-ci-e2e-test resourceVersion: "27913" uid: a02ab100-9464-494b-be82-c23d69aa21c6 kind: Event lastTimestamp: "2026-04-16T17:06:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:06:17Z" name: success-200-isvc-45e11-predictor.18a6e53f209f17a4 namespace: kserve-ci-e2e-test resourceVersion: "28458" uid: 55e3fa46-a8c7-4755-bc17-de03e238d658 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "27662" uid: 30d9a1e8-30e2-495c-8669-70077d6cb45e kind: Event lastTimestamp: "2026-04-16T17:05:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:53Z" name: success-200-isvc-45e11.18a6e531247b74f9 namespace: kserve-ci-e2e-test resourceVersion: "28252" uid: d82a268d-12b6-420e-a2e1-ee2d5a48bb96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "27662" uid: 30d9a1e8-30e2-495c-8669-70077d6cb45e kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-45e11": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-45e11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11.18a6e531250a0496 namespace: kserve-ci-e2e-test resourceVersion: "27684" uid: 3a004821-e526-454b-9716-05c32a5df530 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "27662" uid: 30d9a1e8-30e2-495c-8669-70077d6cb45e kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-45e11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-45e11.18a6e531250a4fc0 namespace: kserve-ci-e2e-test resourceVersion: "27686" uid: 4ff45c8e-5690-4bd2-838f-84d370dd46fc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-45e11 namespace: kserve-ci-e2e-test resourceVersion: "28249" uid: 30d9a1e8-30e2-495c-8669-70077d6cb45e kind: Event lastTimestamp: "2026-04-16T17:05:53Z" message: InferenceService [success-200-isvc-45e11] is Ready metadata: creationTimestamp: "2026-04-16T17:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:53Z" name: success-200-isvc-45e11.18a6e53d2d3a536e namespace: kserve-ci-e2e-test resourceVersion: "28250" uid: fed2ce1d-206e-46be-8aea-ee6da29b7cea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:54:51.914331Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22497" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:54:51Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e4a3084d7825 namespace: kserve-ci-e2e-test resourceVersion: "22502" uid: e95e0d7c-1a59-4c35-a1f3-6372dbdb5ba0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22504" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:54:52Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e4a310f3e7ca namespace: kserve-ci-e2e-test resourceVersion: "22535" uid: 28caf685-7d3f-4f89-a452-2143cbe620ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e4a3128a5f0c namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: 88f2f8e9-e944-4e0c-a3e2-6c2072de24ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e4a31650bd14 namespace: kserve-ci-e2e-test resourceVersion: "22555" uid: 2b648c6b-8c0c-49c3-98ab-b9117938e774 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: "2026-04-16T16:54:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:52Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e4a316ca4114 namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: d97e30a7-bded-44ad-86d5-1eca6edb78ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: "2026-04-16T16:55:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:55:33Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e4a33ec630fa namespace: kserve-ci-e2e-test resourceVersion: "22967" uid: 018f890e-daa7-4e81-b978-b19f619f4f66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: b7366816-b46c-4799-89c3-aecd3144302d kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:16Z" name: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp.18a6e52693425bab namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: d31bc0ac-d6eb-4da4-8a1c-d0057a018c76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-64f8d-predictor-6bdf696c95 namespace: kserve-ci-e2e-test resourceVersion: "22488" uid: 27951897-cdc0-4c76-994b-3ca7ac65e488 kind: Event lastTimestamp: "2026-04-16T16:54:51Z" message: 'Created pod: success-200-isvc-64f8d-predictor-6bdf696c95-hz4cp' metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:51Z" name: success-200-isvc-64f8d-predictor-6bdf696c95.18a6e4a307fc3b73 namespace: kserve-ci-e2e-test resourceVersion: "22499" uid: 39a76cf2-7fc4-4023-8108-1ff9e3342b8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22482" uid: eb08dc76-c1b0-469f-9466-1e0b62cd5853 kind: Event lastTimestamp: "2026-04-16T16:54:51Z" message: Scaled up replica set success-200-isvc-64f8d-predictor-6bdf696c95 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:51Z" name: success-200-isvc-64f8d-predictor.18a6e4a3063c4097 namespace: kserve-ci-e2e-test resourceVersion: "22491" uid: 4723b67c-2642-4077-a089-99dcd2b0511c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22487" uid: 81c3a4ec-1e38-4ea2-b7e8-9e53e62e3c4b kind: Event lastTimestamp: "2026-04-16T16:55:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:36Z" name: success-200-isvc-64f8d-predictor.18a6e4a684d17351 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 610bdc0e-6abb-4f7d-a921-0937e7de8133 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22487" uid: 81c3a4ec-1e38-4ea2-b7e8-9e53e62e3c4b kind: Event lastTimestamp: "2026-04-16T16:55:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:36Z" name: success-200-isvc-64f8d-predictor.18a6e4a6851c9193 namespace: kserve-ci-e2e-test resourceVersion: "22998" uid: 7dab4071-ef44-4a96-a447-61b33909d3b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:55:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22700" uid: 81c3a4ec-1e38-4ea2-b7e8-9e53e62e3c4b kind: Event lastTimestamp: "2026-04-16T16:56:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:07Z" name: success-200-isvc-64f8d-predictor.18a6e4b10438aefb namespace: kserve-ci-e2e-test resourceVersion: "23279" uid: 99980e63-8b4c-4282-b998-6729cb141ac4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:55:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64f8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "22700" uid: 81c3a4ec-1e38-4ea2-b7e8-9e53e62e3c4b kind: Event lastTimestamp: "2026-04-16T16:56:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:56:07Z" name: success-200-isvc-64f8d-predictor.18a6e4b104f9a870 namespace: kserve-ci-e2e-test resourceVersion: "23280" uid: 09194b68-ec63-4293-8cce-67599995ef5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "22469" uid: 4510b282-6960-44f6-b784-125a31738c62 kind: Event lastTimestamp: "2026-04-16T16:55:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:43Z" name: success-200-isvc-64f8d.18a6e4a305f6cc5e namespace: kserve-ci-e2e-test resourceVersion: "23060" uid: c1d1bc61-c397-4937-b3d6-f0a82e66d901 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "22469" uid: 4510b282-6960-44f6-b784-125a31738c62 kind: Event lastTimestamp: "2026-04-16T16:54:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-64f8d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-64f8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:51Z" name: success-200-isvc-64f8d.18a6e4a3069d3e4e namespace: kserve-ci-e2e-test resourceVersion: "22493" uid: cd85f71e-b832-40f6-9195-410073579189 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "22469" uid: 4510b282-6960-44f6-b784-125a31738c62 kind: Event lastTimestamp: "2026-04-16T16:54:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-64f8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:51Z" name: success-200-isvc-64f8d.18a6e4a3069da7c3 namespace: kserve-ci-e2e-test resourceVersion: "22495" uid: 494e044f-4696-4d07-99a2-71614a15758c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64f8d namespace: kserve-ci-e2e-test resourceVersion: "23058" uid: 4510b282-6960-44f6-b784-125a31738c62 kind: Event lastTimestamp: "2026-04-16T16:55:43Z" message: InferenceService [success-200-isvc-64f8d] is Ready metadata: creationTimestamp: "2026-04-16T16:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:43Z" name: success-200-isvc-64f8d.18a6e4af238e5bc6 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: d57d9990-4b6c-414d-be5c-e7a712e1d171 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:41:56.686531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9c599-predictor-546d788cd9-7c769 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:41:56Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3ee891ffff2 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 935037d7-b851-475d-91f7-5cdcaa52667e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:41:56Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3ee90e52318 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 4d88fd40-47d2-4311-817e-be45dc750328 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:41:56Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3ee93110933 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: e92cd7cc-d26d-4cba-a4cc-4556cf41cb15 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:42:08Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" in 11.565s (11.565s including waiting). Image size: 1333164653 bytes.' metadata: creationTimestamp: "2026-04-16T16:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:08Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3f144717a6f namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: c03295d8-1b3b-4c07-b5db-f899c4272164 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:42:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:08Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3f14b5456df namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 0d7f15e6-09d4-49e1-b6c7-499498d5e2ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:42:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:08Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3f14bd98d94 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 7ade952e-4d9b-4a33-9cc9-5779e64ddda6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:42:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:42:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:42:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:42:50Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e3f17585a21a namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 86ae252c-61cc-4586-b50d-e39f14e9049d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c599-predictor-546d788cd9-7c769 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0d425764-a1b0-434e-a0a7-c85340787042 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-9c599-predictor-546d788cd9-7c769.18a6e404ae7ea40d namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: d6d0689b-4628-4369-b172-7a9b0685b7d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9c599-predictor-546d788cd9 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 37e7e6bb-f686-47e6-94fe-d7717346ac34 kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: 'Created pod: success-200-isvc-9c599-predictor-546d788cd9-7c769' metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: success-200-isvc-9c599-predictor-546d788cd9.18a6e3ee8830071a namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 3020b29e-b36e-475f-849e-fdb266e6cd32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: e555490a-51b5-434e-8f47-b3069ec8e08e kind: Event lastTimestamp: "2026-04-16T16:41:56Z" message: Scaled up replica set success-200-isvc-9c599-predictor-546d788cd9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:41:56Z" name: success-200-isvc-9c599-predictor.18a6e3ee81e282ae namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 34e60c8e-a9b6-4a70-be6b-cae01c0a3585 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: 8bd30235-cba5-44b2-aff2-ffa05f17f955 kind: Event lastTimestamp: "2026-04-16T16:42:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:41Z" name: success-200-isvc-9c599-predictor.18a6e3f20066e90a namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 42d212d9-6854-41e9-ad4a-e2f4cc756783 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: 8bd30235-cba5-44b2-aff2-ffa05f17f955 kind: Event lastTimestamp: "2026-04-16T16:42:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:42:41Z" name: success-200-isvc-9c599-predictor.18a6e3f200d30dc6 namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 27a3d83c-3bc8-486c-92a4-b7a829f51062 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:42:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 8bd30235-cba5-44b2-aff2-ffa05f17f955 kind: Event lastTimestamp: "2026-04-16T16:43:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:26Z" name: success-200-isvc-9c599-predictor.18a6e3fc7d8e5ad7 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: e8161619-af43-42ec-87f9-1f7844f882b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:42:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c599-predictor namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 8bd30235-cba5-44b2-aff2-ffa05f17f955 kind: Event lastTimestamp: "2026-04-16T16:43:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:26Z" name: success-200-isvc-9c599-predictor.18a6e3fc7e0075fa namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 7d074cba-448b-4ae9-8bbb-118ea6044b3f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:41:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c599 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 93c800b9-4e9f-43f6-9c6e-e1bde3905eee kind: Event lastTimestamp: "2026-04-16T16:43:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:00Z" name: success-200-isvc-9c599.18a6e3ee81ba49a0 namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: b105eb8f-9b49-4b8e-a497-60f02675ba26 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c599 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 93c800b9-4e9f-43f6-9c6e-e1bde3905eee kind: Event lastTimestamp: "2026-04-16T16:43:00Z" message: InferenceService [success-200-isvc-9c599] is Ready metadata: creationTimestamp: "2026-04-16T16:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:00Z" name: success-200-isvc-9c599.18a6e3fd59ecbccf namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 98af7c30-5429-4eb9-a514-7eaed663ae08 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:04:16.911293Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:04:16Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e52694bf0ac1 namespace: kserve-ci-e2e-test resourceVersion: "27148" uid: 6dd155b4-300b-42ea-a3a6-bee29d86592d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:04:17Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e5269d0991d8 namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: 06884e6a-e3fd-475c-946b-74c3e29073ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:17Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e5269e5ea56c namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: 5e10ba3e-a5e6-45dc-a20f-c55dde0036a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:17Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e526a235bbe1 namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: 2e0b9fcf-0f52-4567-b8aa-63296b8dd5b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: "2026-04-16T17:04:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:04:17Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e526a2af3b6e namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: ff8befac-bf28-424a-b667-da1142264f60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: "2026-04-16T17:21:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:21:48Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e526c564a924 namespace: kserve-ci-e2e-test resourceVersion: "34926" uid: 5007246e-87f2-4440-9892-1ee5728c29e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 92833bc1-eea4-4514-a009-9753726ecfd4 kind: Event lastTimestamp: "2026-04-16T17:21:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:21:46Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd.18a6e61afd106647 namespace: kserve-ci-e2e-test resourceVersion: "34867" uid: 1435f1bc-af36-4506-95cf-159d611935ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b9f91-predictor-bf89fb6c7 namespace: kserve-ci-e2e-test resourceVersion: "27126" uid: 6f5023e8-40af-4f80-af5f-cbf48ecbca63 kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: 'Created pod: success-200-isvc-b9f91-predictor-bf89fb6c7-rjfsd' metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:16Z" name: success-200-isvc-b9f91-predictor-bf89fb6c7.18a6e52694522acd namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: 50b9c8e7-ccb2-4d64-a525-7ba80339efc1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: 3c89bb76-d14d-4e16-a85d-39d7b257f311 kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: Scaled up replica set success-200-isvc-b9f91-predictor-bf89fb6c7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:16Z" name: success-200-isvc-b9f91-predictor.18a6e52692213445 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: f1e3c29c-4a9b-4814-ae3e-7c5d33648001 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T17:04:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 3ec31828-8c43-4e4a-89af-c83a9e752371 kind: Event lastTimestamp: "2026-04-16T17:05:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:31Z" name: success-200-isvc-b9f91-predictor.18a6e52a10ab9b84 namespace: kserve-ci-e2e-test resourceVersion: "28052" uid: 665784c0-6cbe-4d32-806f-5a82d8514ea0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T17:04:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 3ec31828-8c43-4e4a-89af-c83a9e752371 kind: Event lastTimestamp: "2026-04-16T17:05:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:31Z" name: success-200-isvc-b9f91-predictor.18a6e52a111663ce namespace: kserve-ci-e2e-test resourceVersion: "28053" uid: 5598eff9-9cf8-4e7f-940b-40f963287089 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:05:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27338" uid: 3ec31828-8c43-4e4a-89af-c83a9e752371 kind: Event lastTimestamp: "2026-04-16T17:05:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:16Z" name: success-200-isvc-b9f91-predictor.18a6e5310e9730ff namespace: kserve-ci-e2e-test resourceVersion: "27904" uid: 0ccadd9c-9d7f-45b0-976f-b9549c0b403e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:05:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9f91-predictor namespace: kserve-ci-e2e-test resourceVersion: "27338" uid: 3ec31828-8c43-4e4a-89af-c83a9e752371 kind: Event lastTimestamp: "2026-04-16T17:05:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:16Z" name: success-200-isvc-b9f91-predictor.18a6e5310f049f08 namespace: kserve-ci-e2e-test resourceVersion: "27905" uid: 05af0074-8b14-420e-8217-fc10117ee758 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 5748c22d-0056-40db-afca-2ab645be1116 kind: Event lastTimestamp: "2026-04-16T17:05:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:08Z" name: success-200-isvc-b9f91.18a6e5269205c35e namespace: kserve-ci-e2e-test resourceVersion: "27843" uid: e33faf62-47ac-4f16-90cc-da826c051205 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 5748c22d-0056-40db-afca-2ab645be1116 kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-b9f91": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b9f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:16Z" name: success-200-isvc-b9f91.18a6e52692bd7504 namespace: kserve-ci-e2e-test resourceVersion: "27137" uid: d0f36080-1f38-4c26-a694-2ff4c00653a6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 5748c22d-0056-40db-afca-2ab645be1116 kind: Event lastTimestamp: "2026-04-16T17:04:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b9f91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:16Z" name: success-200-isvc-b9f91.18a6e52692bddb4d namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 40976084-ed71-42dd-b324-034ac243393c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27841" uid: 5748c22d-0056-40db-afca-2ab645be1116 kind: Event lastTimestamp: "2026-04-16T17:05:08Z" message: InferenceService [success-200-isvc-b9f91] is Ready metadata: creationTimestamp: "2026-04-16T17:05:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:08Z" name: success-200-isvc-b9f91.18a6e532a9832727 namespace: kserve-ci-e2e-test resourceVersion: "27842" uid: 9babfbe2-958c-4047-a9bc-402b9c8f6e03 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:52:56.991889Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bab33-predictor-556678b97c-f2kql to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:52:56Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e48846642afb namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 8f22f9ba-0746-4003-9cbb-97ada98d7e7e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:52:57Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e4884db14df8 namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 6f58df45-a493-403e-b910-b923a4b77dd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:57Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e4884fd68f60 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 22039756-8856-4038-b767-a05d666a772f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:57Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e48854ebe7e2 namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: f12f0ad7-b1c6-46d0-999c-0f99e50a8c63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: "2026-04-16T16:52:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:57Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e48855700087 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 14dd4432-6b37-40e3-bfe8-810f8c8b4423 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T16:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: "2026-04-16T16:54:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:18Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e4885fc7712b namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 9ec74b5c-e634-4b15-b75a-997ea155c9ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bab33-predictor-556678b97c-f2kql namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 7bf148ef-251a-42b6-ad49-1165aab7abd9 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-bab33-predictor-556678b97c-f2kql.18a6e49af0f98da3 namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: 9493eef7-9365-4d25-8983-7f6c09e127a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bab33-predictor-556678b97c namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 7d6ceef3-75ae-404c-8c6b-5bd7adfb1047 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: 'Created pod: success-200-isvc-bab33-predictor-556678b97c-f2kql' metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:52:56Z" name: success-200-isvc-bab33-predictor-556678b97c.18a6e4884598cf4a namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 8d7f39e2-5532-46e2-8edb-e6c96337cfaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bab33-predictor namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: bd414410-f9f3-424d-9d58-e7f15521b780 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: Scaled up replica set success-200-isvc-bab33-predictor-556678b97c from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:52:56Z" name: success-200-isvc-bab33-predictor.18a6e48839feeab4 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: 893eeb16-1933-426a-9267-1bbe3b297110 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:53:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bab33-predictor namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: b940dbc2-949a-4b64-920f-fa0532537eb3 kind: Event lastTimestamp: "2026-04-16T16:54:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:11Z" name: success-200-isvc-bab33-predictor.18a6e48bb851383f namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: c111898e-4c5c-466e-8fba-ee7f5a712bbe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:53:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bab33-predictor namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: b940dbc2-949a-4b64-920f-fa0532537eb3 kind: Event lastTimestamp: "2026-04-16T16:54:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:11Z" name: success-200-isvc-bab33-predictor.18a6e48bb8c13727 namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: ff268685-84e1-4e0d-ab1b-1ee0d1d4c765 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: ce897980-7a23-448c-9df6-3928f893b899 kind: Event lastTimestamp: "2026-04-16T16:53:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:48Z" name: success-200-isvc-bab33.18a6e48839989d65 namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: daf71eff-b12f-4fd0-99c0-12f4028b116c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: ce897980-7a23-448c-9df6-3928f893b899 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-bab33": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bab33": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:52:56Z" name: success-200-isvc-bab33.18a6e4883a90056a namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 235fb999-96ac-4b2a-abdf-10165ba1a29d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: ce897980-7a23-448c-9df6-3928f893b899 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bab33": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:52:56Z" name: success-200-isvc-bab33.18a6e4883a904426 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 9399849f-7ec7-42f7-bd2f-dd662b89dada reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:53:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bab33 namespace: kserve-ci-e2e-test resourceVersion: "21696" uid: ce897980-7a23-448c-9df6-3928f893b899 kind: Event lastTimestamp: "2026-04-16T16:53:48Z" message: InferenceService [success-200-isvc-bab33] is Ready metadata: creationTimestamp: "2026-04-16T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:53:48Z" name: success-200-isvc-bab33.18a6e49444e51cbb namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 7f5c11a1-110a-4bba-9a9f-9e426e62371a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:54:17.209828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e49af3c0ce75 namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: 0ebd2e50-baf8-49c3-a83a-5b94c1397d8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e49afb6940d8 namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 541f7415-dc52-4e9a-b65d-1c5c46dd9b2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e49afdde1a84 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: 57b77e9b-4b36-4fec-a169-b070f6c1c7f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e49b014a8c68 namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: fd930dd1-84f5-4f4b-aa37-a47dfa797853 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e49b01c2ba74 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: b4aa1b0c-c112-47fb-a799-4c5015cb68be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: "2026-04-16T16:54:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:54:58Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e49b115647c7 namespace: kserve-ci-e2e-test resourceVersion: "22627" uid: ba9f7790-4c21-4f6b-b02b-3a197523209e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7 namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: c6ad7b17-f0cb-4a25-a648-a2c17c4b1205 kind: Event lastTimestamp: "2026-04-16T17:03:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:41Z" name: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7.18a6e51e71378880 namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: 4c0a9fe6-4204-4abd-b13b-7822054d56d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-da571-predictor-d95c9c8b7 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 2a77edcb-89a3-4a6b-aeed-b09115e967a2 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'Created pod: success-200-isvc-da571-predictor-d95c9c8b7-2qvf7' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor-d95c9c8b7.18a6e49af2aa9b7c namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: dedc0375-8987-4083-b9ce-833e369f8155 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 23878969-62fe-4041-9e40-5bd5ae04a380 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: Scaled up replica set success-200-isvc-da571-predictor-d95c9c8b7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571-predictor.18a6e49aef4a16c3 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 290b920f-789c-44a7-9b34-ae58b251d539 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:54:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 31f2729a-8c6d-4c89-a6c5-d997aa005387 kind: Event lastTimestamp: "2026-04-16T16:55:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:32Z" name: success-200-isvc-da571-predictor.18a6e49e6d977927 namespace: kserve-ci-e2e-test resourceVersion: "22949" uid: ee0c024e-dfd5-42f0-9992-48b805b43ecf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:54:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 31f2729a-8c6d-4c89-a6c5-d997aa005387 kind: Event lastTimestamp: "2026-04-16T16:55:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:32Z" name: success-200-isvc-da571-predictor.18a6e49e6e05687d namespace: kserve-ci-e2e-test resourceVersion: "22950" uid: 31853cda-b28d-4dd4-bcd4-e1af2c1f5739 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:55:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22243" uid: 31f2729a-8c6d-4c89-a6c5-d997aa005387 kind: Event lastTimestamp: "2026-04-16T16:55:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:17Z" name: success-200-isvc-da571-predictor.18a6e4a56bf492e0 namespace: kserve-ci-e2e-test resourceVersion: "22800" uid: 5eda2fde-a1ee-400a-8400-517a3fa4cea7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:55:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da571-predictor namespace: kserve-ci-e2e-test resourceVersion: "22243" uid: 31f2729a-8c6d-4c89-a6c5-d997aa005387 kind: Event lastTimestamp: "2026-04-16T16:55:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:55:17Z" name: success-200-isvc-da571-predictor.18a6e4a56c6dfeb8 namespace: kserve-ci-e2e-test resourceVersion: "22801" uid: 966e895d-4cf5-40bf-8336-eebdb2aa0ad5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 1a68c876-6870-408a-87b5-b651f5766482 kind: Event lastTimestamp: "2026-04-16T16:55:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:08Z" name: success-200-isvc-da571.18a6e49aeee9f3f8 namespace: kserve-ci-e2e-test resourceVersion: "22725" uid: 8e80910b-f9d7-4531-ba2b-efce0c0dff28 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 1a68c876-6870-408a-87b5-b651f5766482 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'Failed to update status for InferenceService "success-200-isvc-da571": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-da571": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571.18a6e49af03133e5 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 995c6d38-6880-4ac7-abae-0336f7292be4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 1a68c876-6870-408a-87b5-b651f5766482 kind: Event lastTimestamp: "2026-04-16T16:54:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-da571": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:54:17Z" name: success-200-isvc-da571.18a6e49af031789c namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: b844ca7a-f8cb-48bd-83b9-7caa18dc80ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:55:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da571 namespace: kserve-ci-e2e-test resourceVersion: "22722" uid: 1a68c876-6870-408a-87b5-b651f5766482 kind: Event lastTimestamp: "2026-04-16T16:55:08Z" message: InferenceService [success-200-isvc-da571] is Ready metadata: creationTimestamp: "2026-04-16T16:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:55:08Z" name: success-200-isvc-da571.18a6e4a6f58145d1 namespace: kserve-ci-e2e-test resourceVersion: "22723" uid: 3cac36db-af98-4e0d-82ba-712fe0dcab26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:43:31.848316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-df178-predictor-758f86b6c9-r4h2h to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e404b135b0c7 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 7ea81b0f-1cee-4356-8e0c-9d1e3accf76c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e404b96413ea namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: 27b3f22c-b273-4389-8271-8d4448a4730e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e404bae45a50 namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 533b3160-5f93-4831-9028-24621ba36803 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e404c250dba3 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: 16394dac-b4f1-4e77-9290-870e0e792485 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: "2026-04-16T16:43:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:32Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e404c2cbc065 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 7da38a0c-1cc9-4363-a4e4-595a04ba291f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: "2026-04-16T16:44:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T16:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:13Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e404d90cf98c namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: 1fffa6ab-f57c-426a-aa76-d846466229f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: f100a932-f599-4d75-b479-7e63d1d0dcbc kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:56Z" name: success-200-isvc-df178-predictor-758f86b6c9-r4h2h.18a6e488389ee6c4 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 24963c6d-b1fd-43da-848a-3f598e4f01a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-df178-predictor-758f86b6c9 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 7a26effe-19f0-404d-a2ca-106075265aa5 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'Created pod: success-200-isvc-df178-predictor-758f86b6c9-r4h2h' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-df178-predictor-758f86b6c9.18a6e404b0c59e9f namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 030860c6-6c39-4a16-9338-1bb524ad891a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 63ac9bc0-a2f1-4cf3-8dcd-0256a68e1eb6 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: Scaled up replica set success-200-isvc-df178-predictor-758f86b6c9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-df178-predictor.18a6e404ad55c462 namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: f01baefd-99ef-467f-a82e-0ecea8e58eda reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: 84158f8e-efb6-4dad-9a0d-3bf2dfa9525a kind: Event lastTimestamp: "2026-04-16T16:44:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:01Z" name: success-200-isvc-df178-predictor.18a6e4082c2d5a76 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: dd7c7331-8f5f-4ecd-ac8a-777412de8944 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: 84158f8e-efb6-4dad-9a0d-3bf2dfa9525a kind: Event lastTimestamp: "2026-04-16T16:44:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:01Z" name: success-200-isvc-df178-predictor.18a6e4082c92cd79 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: 0ffbba57-bc82-4eae-a6c1-31631a324e8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 84158f8e-efb6-4dad-9a0d-3bf2dfa9525a kind: Event lastTimestamp: "2026-04-16T16:44:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:46Z" name: success-200-isvc-df178-predictor.18a6e40f2a4282b3 namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: c32d095d-b164-4af8-b43d-945dcef8245c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T16:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df178-predictor namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 84158f8e-efb6-4dad-9a0d-3bf2dfa9525a kind: Event lastTimestamp: "2026-04-16T16:44:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:46Z" name: success-200-isvc-df178-predictor.18a6e40f2aacc3ab namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: 68ba5e60-ad54-46f5-a6cb-3099217fdce6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 3253c0f6-af64-49ce-bb4c-9e408788179d kind: Event lastTimestamp: "2026-04-16T16:44:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:23Z" name: success-200-isvc-df178.18a6e404ad600ded namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 087107be-72c5-429e-bf10-b2e7ba63484b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 3253c0f6-af64-49ce-bb4c-9e408788179d kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-df178": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df178": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-df178.18a6e404ae16beaf namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: 5a77659c-28eb-40c9-8709-159252f5426c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 3253c0f6-af64-49ce-bb4c-9e408788179d kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df178": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:31Z" name: success-200-isvc-df178.18a6e404ae170980 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: cb19387e-7a6e-4b97-9f47-d8140a3233c2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df178 namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: 3253c0f6-af64-49ce-bb4c-9e408788179d kind: Event lastTimestamp: "2026-04-16T16:44:23Z" message: InferenceService [success-200-isvc-df178] is Ready metadata: creationTimestamp: "2026-04-16T16:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:23Z" name: success-200-isvc-df178.18a6e410bd845257 namespace: kserve-ci-e2e-test resourceVersion: "17397" uid: b75e035d-1c01-4ec2-a60c-4998226f0f51 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:03:42.061346Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-df533-predictor-6bc8658849-wmfj8 to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:03:42Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e51e7786eee8 namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: b326c8d1-0879-4ac4-a723-51a786e0e720 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26778" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:03:42Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e51e80050ca6 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: aaf23439-b070-4499-9948-5a232587bb60 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1324" already present on machine metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e51e813eb0ca namespace: kserve-ci-e2e-test resourceVersion: "26802" uid: 759d175f-1234-4ee8-bba9-9d639f5d1e12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e51e87e793ae namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: b8949345-28e5-4c39-ab87-8326fdf9b688 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:03:42Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e51e885ea808 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: edb4bef2-fe3d-4973-a54f-1a091bd6e55f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: "2026-04-16T17:05:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:03Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e51e97ece17e namespace: kserve-ci-e2e-test resourceVersion: "27777" uid: fdd221d9-ffcf-49ad-9286-ea4d89503ae0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df533-predictor-6bc8658849-wmfj8 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: ace6f6ff-6b2e-4aec-bde2-9007ab3736d0 kind: Event lastTimestamp: "2026-04-16T17:05:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:02Z" name: success-200-isvc-df533-predictor-6bc8658849-wmfj8.18a6e531273bd41a namespace: kserve-ci-e2e-test resourceVersion: "27695" uid: c07b521a-5cb4-4b1d-a55b-001285ec9fde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-df533-predictor-6bc8658849 namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: a3b31d2f-7335-4ca5-9dda-8fb0a57be854 kind: Event lastTimestamp: "2026-04-16T17:03:42Z" message: 'Created pod: success-200-isvc-df533-predictor-6bc8658849-wmfj8' metadata: creationTimestamp: "2026-04-16T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:42Z" name: success-200-isvc-df533-predictor-6bc8658849.18a6e51e7629b937 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: fe5caf95-13af-48d9-8277-a44a7a5d0ea4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-df533-predictor namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: 60df247c-6042-42c6-92a5-a401d1a01093 kind: Event lastTimestamp: "2026-04-16T17:03:41Z" message: Scaled up replica set success-200-isvc-df533-predictor-6bc8658849 from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:03:41Z" name: success-200-isvc-df533-predictor.18a6e51e717b7a00 namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: c5bcdce8-36b2-4160-872e-9c5842246373 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:03:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df533-predictor namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: a5f4f192-047c-48e3-acb1-d469bddf8bc3 kind: Event lastTimestamp: "2026-04-16T17:04:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:42Z" name: success-200-isvc-df533-predictor.18a6e521f018fcbc namespace: kserve-ci-e2e-test resourceVersion: "27434" uid: 398f3e52-6c71-4988-a9f5-eaef3b50923d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T17:03:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df533-predictor namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: a5f4f192-047c-48e3-acb1-d469bddf8bc3 kind: Event lastTimestamp: "2026-04-16T17:04:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T17:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:04:42Z" name: success-200-isvc-df533-predictor.18a6e521f079b108 namespace: kserve-ci-e2e-test resourceVersion: "27435" uid: 50295cbc-6a22-45b5-82d5-58a486856423 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T17:03:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 5462d60d-b8ad-48fa-a423-6df0c8d7dccb kind: Event lastTimestamp: "2026-04-16T17:04:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:33Z" name: success-200-isvc-df533.18a6e51e71687a60 namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: a33cb9f4-6bbd-42f7-a2bc-099d35a53d80 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 5462d60d-b8ad-48fa-a423-6df0c8d7dccb kind: Event lastTimestamp: "2026-04-16T17:03:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-df533": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df533": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:41Z" name: success-200-isvc-df533.18a6e51e72503ad3 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: c107fbc3-0819-491a-be20-4f6a718243dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:03:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 5462d60d-b8ad-48fa-a423-6df0c8d7dccb kind: Event lastTimestamp: "2026-04-16T17:03:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df533": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:03:41Z" name: success-200-isvc-df533.18a6e51e7250876e namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 6f8b4e83-5dd1-443c-ac0b-9b449a011fc8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:04:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df533 namespace: kserve-ci-e2e-test resourceVersion: "27370" uid: 5462d60d-b8ad-48fa-a423-6df0c8d7dccb kind: Event lastTimestamp: "2026-04-16T17:04:33Z" message: InferenceService [success-200-isvc-df533] is Ready metadata: creationTimestamp: "2026-04-16T17:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:04:33Z" name: success-200-isvc-df533.18a6e52a7c5d0e8e namespace: kserve-ci-e2e-test resourceVersion: "27371" uid: fb0147ac-c632-4dcf-85a6-72d52b77da2c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:43:16.482764Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9c599-57c9898f45-ckpsz to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:43:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:43:16Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e4011d59e4af namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: b1ec40db-dafd-42fd-9744-9e8ae2c0f36b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9c599-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T16:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:16Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e4012b7f3331 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: ba315666-691a-4a07-98a2-61e9466eefe9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:17Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:43:17Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e4015a416dbe namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 97ae562b-c44a-4ab7-829b-7d3a258821ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c599} kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:17Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" metadata: creationTimestamp: "2026-04-16T16:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:17Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e4015c0ee34a namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 055bdeae-c30a-4297-a129-38542dfa6a6c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c599} kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" in 2.234s (2.234s including waiting). Image size: 216206422 bytes.' metadata: creationTimestamp: "2026-04-16T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:19Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e401e13e367d namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 3f4be2de-9849-4aab-aa6d-a41198ee1a7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c599} kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:19Z" message: 'Created container: switch-graph-9c599' metadata: creationTimestamp: "2026-04-16T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:19Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e401e43ea6c0 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: e6938213-95c3-4d50-94ce-09233681a8bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c599} kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:19Z" message: Started container switch-graph-9c599 metadata: creationTimestamp: "2026-04-16T16:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:19Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e401e4b59055 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 8d48fc13-62c4-4710-813f-5a806bbd811e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c599} kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: Stopping container switch-graph-9c599 metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:43:31Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e404a5b83fd0 namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 4533f053-7ba2-4599-8c3c-872c00a27f95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c599} kind: Pod name: switch-graph-9c599-57c9898f45-ckpsz namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 269dbbe4-1334-42a2-a926-2b64a2566a66 kind: Event lastTimestamp: "2026-04-16T16:44:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:01Z" name: switch-graph-9c599-57c9898f45-ckpsz.18a6e405c5815ee3 namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 7a723011-c32f-47a8-b7a0-420063d392dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9c599-57c9898f45 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: bd528743-1fb6-4976-89ad-c7386ecdf4c4 kind: Event lastTimestamp: "2026-04-16T16:43:16Z" message: 'Created pod: switch-graph-9c599-57c9898f45-ckpsz' metadata: creationTimestamp: "2026-04-16T16:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:16Z" name: switch-graph-9c599-57c9898f45.18a6e4011c2aa79f namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: b41cb3db-627e-4649-b6ff-f106140fd9aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9c599 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: c8869c98-0767-4ef9-a854-c35307d1d5f2 kind: Event lastTimestamp: "2026-04-16T16:43:16Z" message: Scaled up replica set switch-graph-9c599-57c9898f45 from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:16Z" name: switch-graph-9c599.18a6e40119ae8c14 namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: fc3897fe-c595-474a-9b96-a18e3a3a81a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9c599 namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: d0dd2197-f306-40c2-80fa-ccb4e82226d5 kind: Event lastTimestamp: "2026-04-16T16:43:16Z" message: InferenceGraph [switch-graph-9c599] is Ready metadata: creationTimestamp: "2026-04-16T16:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:43:16Z" name: switch-graph-9c599.18a6e4012295c442 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 73ffa6e0-84c6-4870-aaa9-266633637b90 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c599 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: a6de4bee-8d5e-4554-8769-80cecf756cb0 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:31Z" name: switch-graph-9c599.18a6e40497acd150 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: bf013cfb-6969-4f2e-a47b-dde7f4ee6d5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c599 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: a6de4bee-8d5e-4554-8769-80cecf756cb0 kind: Event lastTimestamp: "2026-04-16T16:43:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T16:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:43:31Z" name: switch-graph-9c599.18a6e40498222e38 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: 4f4df90c-425d-476f-985d-508013ce1109 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T17:05:26.992553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27992" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b9f91-76696f6fcb-nw6hn to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T17:05:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T17:05:26Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e536e5ea1a34 namespace: kserve-ci-e2e-test resourceVersion: "27997" uid: 0ae24a55-584b-401d-bdec-f90e3655211a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:05:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-b9f91-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T17:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:27Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e536efa7c236 namespace: kserve-ci-e2e-test resourceVersion: "28009" uid: 620ec68e-8585-4c9a-8dba-c48f69bdcf7b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:28Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "28001" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:05:28Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T17:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T17:05:28Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e53722bff968 namespace: kserve-ci-e2e-test resourceVersion: "28018" uid: 749142c4-1b44-4959-a035-31e0642be141 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9f91} kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:05:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T17:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:28Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e537243d3b0a namespace: kserve-ci-e2e-test resourceVersion: "28020" uid: bb83e5f0-97b1-4ee2-9f87-7ee6bfbf466d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9f91} kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:05:28Z" message: 'Created container: switch-graph-b9f91' metadata: creationTimestamp: "2026-04-16T17:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:28Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e53727544273 namespace: kserve-ci-e2e-test resourceVersion: "28021" uid: 573d430c-f4f9-4fbb-8d63-a30c8fa477c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9f91} kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:05:28Z" message: Started container switch-graph-b9f91 metadata: creationTimestamp: "2026-04-16T17:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:05:28Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e53727c97a9d namespace: kserve-ci-e2e-test resourceVersion: "28022" uid: 92edc30f-3d1d-4ba2-877b-9bcfb82079b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9f91} kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:21:46Z" message: Stopping container switch-graph-b9f91 metadata: creationTimestamp: "2026-04-16T17:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:21:46Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e61af6b2fcb8 namespace: kserve-ci-e2e-test resourceVersion: "34854" uid: a9331e5c-ddf2-4f23-9a13-a0fb78dac327 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T17:21:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b9f91} kind: Pod name: switch-graph-b9f91-76696f6fcb-nw6hn namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: d695fc0a-51a9-4ef8-91f6-db4665e6a315 kind: Event lastTimestamp: "2026-04-16T17:22:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T17:21:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T17:22:09Z" name: switch-graph-b9f91-76696f6fcb-nw6hn.18a6e61bc4d6927b namespace: kserve-ci-e2e-test resourceVersion: "35093" uid: 5cbbbf6d-e601-4e79-bc3f-48205cca498b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b9f91-76696f6fcb namespace: kserve-ci-e2e-test resourceVersion: "27989" uid: 55ac75d3-8723-4929-a145-8b9004c4c439 kind: Event lastTimestamp: "2026-04-16T17:05:26Z" message: 'Created pod: switch-graph-b9f91-76696f6fcb-nw6hn' metadata: creationTimestamp: "2026-04-16T17:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:26Z" name: switch-graph-b9f91-76696f6fcb.18a6e536e588868a namespace: kserve-ci-e2e-test resourceVersion: "27993" uid: 18b3750a-0bae-4532-8f68-4a161b9096be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27987" uid: 3a2db42c-67f8-4b8f-93f5-6d109b6755e5 kind: Event lastTimestamp: "2026-04-16T17:05:26Z" message: Scaled up replica set switch-graph-b9f91-76696f6fcb from 0 to 1 metadata: creationTimestamp: "2026-04-16T17:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:26Z" name: switch-graph-b9f91.18a6e536e4cc0a3f namespace: kserve-ci-e2e-test resourceVersion: "27990" uid: 8a2375f3-218d-451b-825b-b045510b2a08 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T17:05:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "28003" uid: a1586a10-a59f-4543-9d4e-7fe488ad5c02 kind: Event lastTimestamp: "2026-04-16T17:05:27Z" message: InferenceGraph [switch-graph-b9f91] is Ready metadata: creationTimestamp: "2026-04-16T17:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T17:05:27Z" name: switch-graph-b9f91.18a6e536e864c62d namespace: kserve-ci-e2e-test resourceVersion: "28005" uid: f4b642e4-c971-40e4-b67e-3404adc1ecf2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:05:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27988" uid: 0d579d43-c56d-4c4a-b04d-57d55c9dd20e kind: Event lastTimestamp: "2026-04-16T17:05:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:57Z" name: switch-graph-b9f91.18a6e53a63418327 namespace: kserve-ci-e2e-test resourceVersion: "28276" uid: bbd7f2ed-e7c3-4d84-aad9-e33081e3616b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T17:05:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b9f91 namespace: kserve-ci-e2e-test resourceVersion: "27988" uid: 0d579d43-c56d-4c4a-b04d-57d55c9dd20e kind: Event lastTimestamp: "2026-04-16T17:05:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T17:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T17:05:57Z" name: switch-graph-b9f91.18a6e53a63a4a3db namespace: kserve-ci-e2e-test resourceVersion: "28277" uid: a9e18b02-c51d-4f1d-9a2c-e3dabccef397 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T16:44:41.938409Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17550" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-df178-67f9fc49fb-mccss to ip-10-0-128-173.ec2.internal metadata: creationTimestamp: "2026-04-16T16:44:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T16:44:41Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e41502e795cd namespace: kserve-ci-e2e-test resourceVersion: "17555" uid: 783634a8-008f-402a-bfff-590044c50e7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79bb4cfc96-rbx2j source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:44:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-df178-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:42Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e4150fc4457e namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: bdbca556-9e08-424c-9446-fa911ec7658c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:44:42Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T16:44:42Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e4153fcfd819 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: 2d3449a6-1561-41e0-a56f-db2f83f4842c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df178} kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:44:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:fa769602a23f523b573db8418e88e5bc10cc9d997f9a762a0e4929807e15e82b" already present on machine metadata: creationTimestamp: "2026-04-16T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:42Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e4154181d473 namespace: kserve-ci-e2e-test resourceVersion: "17578" uid: 43b97db5-92c3-4847-b993-a1cea9cc4bf8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df178} kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:44:43Z" message: 'Created container: switch-graph-df178' metadata: creationTimestamp: "2026-04-16T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:43Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e4154533035b namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: cc4718de-1efe-47a1-90ac-ed3db1af4427 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df178} kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:44:43Z" message: Started container switch-graph-df178 metadata: creationTimestamp: "2026-04-16T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:44:43Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e41545a54224 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: 1916df3c-4b29-4a00-9c38-70aec761c7ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:52:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df178} kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:52:56Z" message: Stopping container switch-graph-df178 metadata: creationTimestamp: "2026-04-16T16:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:52:56Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e4882f74f193 namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: 44030dea-3058-44b7-8c9c-111f02a9ccb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T16:52:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df178} kind: Pod name: switch-graph-df178-67f9fc49fb-mccss namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71258a00-ac83-4668-b057-9eef513ff832 kind: Event lastTimestamp: "2026-04-16T16:53:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T16:52:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T16:53:24Z" name: switch-graph-df178-67f9fc49fb-mccss.18a6e488ec01a8bb namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 4182de41-d1c2-445a-a817-af67f7b5ad6e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-173.ec2.internal source: component: kubelet host: ip-10-0-128-173.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-df178-67f9fc49fb namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: 2e598ed0-1efa-4664-86e0-2e4fd195bf37 kind: Event lastTimestamp: "2026-04-16T16:44:41Z" message: 'Created pod: switch-graph-df178-67f9fc49fb-mccss' metadata: creationTimestamp: "2026-04-16T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:41Z" name: switch-graph-df178-67f9fc49fb.18a6e415025658db namespace: kserve-ci-e2e-test resourceVersion: "17552" uid: fbb1465e-4e7f-486b-a994-6a5183bacb28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-df178 namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 205fa13d-7d7f-4544-a835-12d39ee9b542 kind: Event lastTimestamp: "2026-04-16T16:44:41Z" message: Scaled up replica set switch-graph-df178-67f9fc49fb from 0 to 1 metadata: creationTimestamp: "2026-04-16T16:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:44:41Z" name: switch-graph-df178.18a6e415011c4a50 namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 1935597b-0ab8-45f2-9350-f7461c64dd2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T16:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-df178 namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 556cb9c7-706e-494a-8f31-a2945a7a8ef7 kind: Event lastTimestamp: "2026-04-16T16:44:42Z" message: InferenceGraph [switch-graph-df178] is Ready metadata: creationTimestamp: "2026-04-16T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T16:44:42Z" name: switch-graph-df178.18a6e4150781e70f namespace: kserve-ci-e2e-test resourceVersion: "17564" uid: 9893fa4e-9f89-4961-9f7a-c192567013d5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:44:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-df178 namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: fd665a2b-4d15-4383-9d1e-d6bca7b61c92 kind: Event lastTimestamp: "2026-04-16T16:45:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:11Z" name: switch-graph-df178.18a6e4187fa821d9 namespace: kserve-ci-e2e-test resourceVersion: "17831" uid: 9008bab8-d316-4ae2-bae1-32053b48fb55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T16:44:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-df178 namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: fd665a2b-4d15-4383-9d1e-d6bca7b61c92 kind: Event lastTimestamp: "2026-04-16T16:45:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T16:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T16:45:11Z" name: switch-graph-df178.18a6e4187fec2bb7 namespace: kserve-ci-e2e-test resourceVersion: "17832" uid: caca4c4d-529c-4bfb-93e8-fabea3b4334e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "35097"