--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:16.318777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ac147-5f4c47599d-4q9vq to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41eec931909d namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 1b6cbf7a-a710-49b1-8515-1842bcfaeaa8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20251" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41eee2d9976c namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: d141781e-8bf4-4db3-986e-43acd1fd01c6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac147} kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41eee4247c39 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 8c1a0078-576f-427b-98a0-dae74277f76f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac147} kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Created container: ensemble-graph-ac147' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41eee78faee6 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: f6d52d21-5b23-4b7e-9ce4-0cdb930cb0cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac147} kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Started container ensemble-graph-ac147 metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41eee803537f namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: aa3fa585-b2d9-4e61-a77f-a392793391d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac147} kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Stopping container ensemble-graph-ac147 metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41f11dc5e956 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 62dc1d4f-a5a9-4e37-91ad-07dcc928a499 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ac147} kind: Pod name: ensemble-graph-ac147-5f4c47599d-4q9vq namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 1ba94df5-3413-40d5-834a-f447887dee1e kind: Event lastTimestamp: "2026-07-01T19:43:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:53Z" name: ensemble-graph-ac147-5f4c47599d-4q9vq.18be41f190affb59 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: a2cbc86b-6e21-4398-8fca-270ab38b5721 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ac147-5f4c47599d namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 13cc188a-3493-4ba5-8698-4c914cab3b49 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Created pod: ensemble-graph-ac147-5f4c47599d-4q9vq' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147-5f4c47599d.18be41eec89bbc2d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 496219ee-8717-423e-bce5-a694716f475d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ac147 namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: 04a497c8-5477-45c1-abd5-9b3068c65199 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Scaled up replica set ensemble-graph-ac147-5f4c47599d from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147.18be41eec7e10a28 namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 16f061c9-2809-4265-a693-d3cac502b506 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ac147 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 86952846-55b4-4238-8b9d-ff779549aa07 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: InferenceGraph [ensemble-graph-ac147] is Ready metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147.18be41eecdb681f2 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 260879ca-5365-4856-9223-1b0323c05d97 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ac147 namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 86952846-55b4-4238-8b9d-ff779549aa07 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-ac147": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-ac147": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147.18be41eed0733389 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 7a2f7d3c-568e-448c-b096-9dec2b54df0e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ac147 namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 86952846-55b4-4238-8b9d-ff779549aa07 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-ac147": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:16Z" name: ensemble-graph-ac147.18be41eed0737e69 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: a84c4f47-f5c3-42d4-b965-879ba4b28e85 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:44:26.526026Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c6a40-c7cbdccb5-rgtn5 to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:44:26Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be41ff21df2f74 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: fe478861-7c14-4c4b-bb42-a10829b7f4a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:44:26Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be41ff3b5e8d68 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: e40b7664-d10a-43ab-a7d1-86fa1f9d5031 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6a40} kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be41ff3cb0bfae namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 7f7bb6df-0f98-404f-a2fe-41d3d8a9769a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6a40} kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: 'Created container: ensemble-graph-c6a40' metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be41ff3f87c02a namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: aab10848-e2e5-4c6a-9322-8376f169e4dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6a40} kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Started container ensemble-graph-c6a40 metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be41ff3ff6fe84 namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 752d33bb-e1d6-4cdd-a07b-5ea626f239d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6a40} kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Stopping container ensemble-graph-c6a40 metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be427245bf9b4d namespace: kserve-ci-e2e-test resourceVersion: "24796" uid: 468fc2f1-128a-4e2a-9c18-7be80c152589 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6a40} kind: Pod name: ensemble-graph-c6a40-c7cbdccb5-rgtn5 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: d77279e9-f056-42fc-bf5b-d4a3a951ee0c kind: Event lastTimestamp: "2026-07-01T19:53:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:08Z" name: ensemble-graph-c6a40-c7cbdccb5-rgtn5.18be4272dcc8ff1c namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: a1a8c6cc-91dd-4fc7-9db9-49df45b6b9c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c6a40-c7cbdccb5 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 15e65c36-e212-4bc0-a9d3-cdcbea160a57 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'Created pod: ensemble-graph-c6a40-c7cbdccb5-rgtn5' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:26Z" name: ensemble-graph-c6a40-c7cbdccb5.18be41ff21275ca3 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 781db60f-b4a6-41c6-a88c-3c3532a73d0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 705b51be-8c3b-4cf5-bf7d-0fd82915cb30 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Scaled up replica set ensemble-graph-c6a40-c7cbdccb5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:26Z" name: ensemble-graph-c6a40.18be41ff207491de namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 64bf2c4e-460b-4075-a272-bd1e17cf231b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 2e6192b5-2243-4908-8626-f4a799520d91 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: InferenceGraph [ensemble-graph-c6a40] is Ready metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:26Z" name: ensemble-graph-c6a40.18be41ff263e6603 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: b23fc667-826d-4f8a-acc1-a80994aca77c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: db6c7246-d6a6-4434-a3fa-c4179070b3ac kind: Event lastTimestamp: "2026-07-01T19:45:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:11Z" name: ensemble-graph-c6a40.18be42029ee9ccd3 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: a358a25e-2efc-47b5-a2bc-0cc18bdfe41b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: db6c7246-d6a6-4434-a3fa-c4179070b3ac kind: Event lastTimestamp: "2026-07-01T19:45:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:11Z" name: ensemble-graph-c6a40.18be42029f52c41e namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 941cf6b5-1308-4425-b6d1-d0337898b7b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:34:01.267639Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-01ba2-predictor-66f759c884-26btm to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416d8d91646d namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: c4f71815-6a24-4fd8-88da-d68bf178f3df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416da721471c namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: b56189f6-3db7-4a32-bec6-1c8a280f92f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416da93d0c62 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 49e71891-20ea-4728-90f0-b991216275d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416dabcca344 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: feaf821c-a7ad-4ceb-843a-7e710fffc0bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416dac45d673 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 927eab28-3647-471c-94b9-8e7512ba2f54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416dac500fb8 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: a63ccc67-4e81-4e35-bec0-9d9c9d755e34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416daf3dafcc namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: ce655735-ef87-4ff8-8ffc-bf3ebdf79a0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416dafcd063d namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 499ac156-b53b-4d68-91c2-6a69ae07e918 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:43:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:18Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be416dcec74136 namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: f9258b1f-5cbc-4f79-880e-d6e567e913e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be41eebfa54cce namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 86ca31d9-4154-429e-ab53-933c3eae807d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be41eebfa5bee2 namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 1c5eb026-e13d-42d4-82d7-f752b406c960 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01ba2-predictor-66f759c884-26btm namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 1f8faff5-2165-49ec-9c6b-953905b8abc7 kind: Event lastTimestamp: "2026-07-01T19:43:18Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:18Z" name: error-404-isvc-01ba2-predictor-66f759c884-26btm.18be41ef433e817a namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: ec5fe769-ee4e-472b-91eb-b2b312c42a02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-01ba2-predictor-66f759c884 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 90b9a4b5-ccc7-4ea3-9e4f-cc5fcc628e9e kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'Created pod: error-404-isvc-01ba2-predictor-66f759c884-26btm' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor-66f759c884.18be416d8d1967e7 namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: e6e6ae43-af65-41ec-9478-2d539653f261 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 769e0d5a-0874-48c9-a66e-816321e58105 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Scaled up replica set error-404-isvc-01ba2-predictor-66f759c884 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: error-404-isvc-01ba2-predictor.18be416d89f8a335 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: c9f03ff8-4e3a-48f3-9964-32dfd417227a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 34eec886-90fd-44cf-ac08-581f8c9a5ad8 kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: error-404-isvc-01ba2-predictor.18be4171092b2f6b namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 3511164a-dc9e-4080-b26a-4ca051b87974 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 34eec886-90fd-44cf-ac08-581f8c9a5ad8 kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: error-404-isvc-01ba2-predictor.18be417109ccfde3 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 308dbf0f-5d20-4b7b-bafd-5148519b944f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 34eec886-90fd-44cf-ac08-581f8c9a5ad8 kind: Event lastTimestamp: "2026-07-01T19:35:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:01Z" name: error-404-isvc-01ba2-predictor.18be417807704027 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 441dff84-c249-430c-82b3-4647c3c5b7d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 34eec886-90fd-44cf-ac08-581f8c9a5ad8 kind: Event lastTimestamp: "2026-07-01T19:35:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:01Z" name: error-404-isvc-01ba2-predictor.18be417807e58348 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 181a8b0e-4500-47fc-b836-6e9d471419a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 3f7639a8-4739-4e93-a449-9b32a7567653 kind: Event lastTimestamp: "2026-07-01T19:34:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:48Z" name: error-404-isvc-01ba2.18be416d88e66e76 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: ab7035c0-f79b-4195-a06c-6a4c2e10faa0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: 3f7639a8-4739-4e93-a449-9b32a7567653 kind: Event lastTimestamp: "2026-07-01T19:34:48Z" message: InferenceService [error-404-isvc-01ba2] is Ready metadata: creationTimestamp: "2026-07-01T19:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:48Z" name: error-404-isvc-01ba2.18be41788e0e7b91 namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 302ce99d-97f5-4083-bd8f-a70a2f05e525 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:16.089019Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eebb7fc92c namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 7d622f0f-13a7-44ed-9eed-7595c267ed4b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eed520c8c0 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: a5f3e14a-9da8-406e-8b6b-b1c5d3a7db8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eed6abe93d namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 67d83b87-af99-46dc-b898-2dec9c8bc984 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eed9825910 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: ec5c3274-f729-44ff-a443-6a10b97ecc37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eed9e25a6b namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: c5c95ee1-ecc8-413e-8af7-34dae80eb892 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eed9ee219b namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: b4d4708a-8a9a-4542-9a28-de4c378680a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eedcadc18a namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: beeea01f-1728-4fa3-b681-94adcc3c7e6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eedd0d4123 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: c3163486-1043-43df-91c1-cf6ffdf4b9b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:43:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:52Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41eee2ee537a namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 5b6f5419-9a16-4906-8fa2-5ca9bf19f6f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41ff0d234225 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: fc132de7-ec26-4cb5-bdea-8faf81cdbe6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41ff0d24112f namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 52698841-b93b-48c8-ae2c-49a22333ef5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 617a6c91-9eb5-4902-bd06-aa2916ded127 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc.18be41ff6b21e9f8 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 8baf96bf-8580-402d-999e-1dd3dec42e7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-46ddd-predictor-5cc8f4b6d4 namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: 9e618865-fe22-4d44-84f9-8bdbfb5ecc97 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Created pod: error-404-isvc-46ddd-predictor-5cc8f4b6d4-zd9kc' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor-5cc8f4b6d4.18be41eebb0d8ca4 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: d45b3722-1ddc-44a8-a06d-e0a9e0177e75 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 60a3db2b-e068-4224-8f2f-398944cc3ba2 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: Scaled up replica set error-404-isvc-46ddd-predictor-5cc8f4b6d4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd-predictor.18be41eeb9807335 namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 115bbc29-dc4e-4199-aec6-e542484bd397 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 42a28df8-c48c-4c9f-8090-b9447eae4239 kind: Event lastTimestamp: "2026-07-01T19:43:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:46Z" name: error-404-isvc-46ddd-predictor.18be41f23801477d namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 0c3d6895-2a46-4efb-8a52-341159a941e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 42a28df8-c48c-4c9f-8090-b9447eae4239 kind: Event lastTimestamp: "2026-07-01T19:43:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:46Z" name: error-404-isvc-46ddd-predictor.18be41f23861b4be namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 62d93a4b-b096-4499-9aaa-5594f08f671e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 42a28df8-c48c-4c9f-8090-b9447eae4239 kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:16Z" name: error-404-isvc-46ddd-predictor.18be41f935ce6a2d namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: f4d6e17c-9696-45b7-9ee0-d67c4252b2a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 42a28df8-c48c-4c9f-8090-b9447eae4239 kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:16Z" name: error-404-isvc-46ddd-predictor.18be41f93643204f namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: 930cbb34-32ae-4ee7-af05-e26fa3da9ec9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 27bad6a8-af53-4851-9a26-d713c509a4cf kind: Event lastTimestamp: "2026-07-01T19:44:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:02Z" name: error-404-isvc-46ddd.18be41eeb98a3de8 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 30f35d68-63e2-449b-b4cc-4a4418eb0c0c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 27bad6a8-af53-4851-9a26-d713c509a4cf kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-46ddd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-46ddd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd.18be41eeb9fe8f36 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 7741c5b6-64cc-483e-aeba-e294eddacac2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 27bad6a8-af53-4851-9a26-d713c509a4cf kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-46ddd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-46ddd.18be41eeb9fed5cf namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: faea2bc6-ecc1-4a35-94d4-08a9152160df reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 27bad6a8-af53-4851-9a26-d713c509a4cf kind: Event lastTimestamp: "2026-07-01T19:44:02Z" message: InferenceService [error-404-isvc-46ddd] is Ready metadata: creationTimestamp: "2026-07-01T19:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:02Z" name: error-404-isvc-46ddd.18be41f99e3b93c1 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 95baf9a8-a1f4-40bb-bb62-9b2d821fcc11 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:52:41.372665Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24865" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-74731-predictor-545cd488f6-w9m6r to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be42725907ae13 namespace: kserve-ci-e2e-test resourceVersion: "24870" uid: 5543db93-0188-4f85-bfff-2f1afa35771a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24879" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be427272d7f02d namespace: kserve-ci-e2e-test resourceVersion: "24904" uid: 623c37da-0c97-46ef-b7ad-b8c9a40dba2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be427274355057 namespace: kserve-ci-e2e-test resourceVersion: "24908" uid: 44f9f6b6-b50c-4654-9e9e-b562ca6b0e07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be42727718d6d0 namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: fb4850bb-0f22-4dd0-9e23-abdd998df2a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be4272778b301e namespace: kserve-ci-e2e-test resourceVersion: "24914" uid: e14cceba-c55b-4b4c-a317-e72cddb34902 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be427277957f51 namespace: kserve-ci-e2e-test resourceVersion: "24915" uid: 3a992fec-9b77-4ed9-a98e-e9d7eba45de8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be42727acbaec5 namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: abb5f952-0e8d-4241-9458-c0ce8323b41b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be42727b3c03de namespace: kserve-ci-e2e-test resourceVersion: "24917" uid: f488214f-8642-42a0-9ea3-ca6df5993506 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:53:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:19Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be4272c6529908 namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: fb3cee39-b4b6-42f1-96d0-9692450680d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be4282c84dcb91 namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 89894468-1d5c-4c0f-9a08-c600b8f21e71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be4282c84e4e88 namespace: kserve-ci-e2e-test resourceVersion: "26053" uid: 895052d2-1b2e-4d69-a69f-93f432337842 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-74731-predictor-545cd488f6-w9m6r namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 9c5b3090-df12-47ec-a39d-67714ce7e797 kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:54Z" name: error-404-isvc-74731-predictor-545cd488f6-w9m6r.18be42834e6fc2c5 namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: 422be99e-b07d-46a2-a48c-ac43c584b53b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-74731-predictor-545cd488f6 namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 43db42bc-dfca-46a7-ba1e-24188ba55c9d kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Created pod: error-404-isvc-74731-predictor-545cd488f6-w9m6r' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor-545cd488f6.18be4272588c81a3 namespace: kserve-ci-e2e-test resourceVersion: "24867" uid: 688d059e-d1e3-4eb9-9dfd-2e1f8097f27e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "24852" uid: 31def40b-a75e-415d-840f-5bba33850115 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Scaled up replica set error-404-isvc-74731-predictor-545cd488f6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731-predictor.18be427256a705ed namespace: kserve-ci-e2e-test resourceVersion: "24858" uid: 415e3e14-386e-4cdb-9d23-b41f833b0f1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "24855" uid: 714a8eb8-a605-4f0f-bfbd-82242fab1a30 kind: Event lastTimestamp: "2026-07-01T19:52:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:56Z" name: error-404-isvc-74731-predictor.18be4275d5576876 namespace: kserve-ci-e2e-test resourceVersion: "25039" uid: b3a092a5-d491-426b-8fc5-466de3f06840 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "24855" uid: 714a8eb8-a605-4f0f-bfbd-82242fab1a30 kind: Event lastTimestamp: "2026-07-01T19:52:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:56Z" name: error-404-isvc-74731-predictor.18be4275d5c34cd6 namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: e710173e-a208-4adb-a84a-3f1a218a5771 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: 714a8eb8-a605-4f0f-bfbd-82242fab1a30 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-74731-predictor.18be427954f14973 namespace: kserve-ci-e2e-test resourceVersion: "25517" uid: 76f3b91f-da7b-4c1e-adbc-b435ab25ad02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: 714a8eb8-a605-4f0f-bfbd-82242fab1a30 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-74731-predictor.18be4279555b0af9 namespace: kserve-ci-e2e-test resourceVersion: "25518" uid: 445799a3-7460-418a-8634-fc4671da4d78 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "24823" uid: d955fdcd-8584-4d2a-8705-8d5005e7bf6f kind: Event lastTimestamp: "2026-07-01T19:53:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:29Z" name: error-404-isvc-74731.18be427256dfe62e namespace: kserve-ci-e2e-test resourceVersion: "25306" uid: 075cde16-b0c3-47fd-9a52-c65b0749913c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "24823" uid: d955fdcd-8584-4d2a-8705-8d5005e7bf6f kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-74731": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-74731": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731.18be4272574524fe namespace: kserve-ci-e2e-test resourceVersion: "24862" uid: 1363a4f8-a9c4-4ee0-bef0-335ed2559e01 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "24823" uid: d955fdcd-8584-4d2a-8705-8d5005e7bf6f kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-74731": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-74731.18be427257456bcc namespace: kserve-ci-e2e-test resourceVersion: "24864" uid: 460f2f00-fb0b-4fe5-bd2b-8a70847ea1a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: d955fdcd-8584-4d2a-8705-8d5005e7bf6f kind: Event lastTimestamp: "2026-07-01T19:53:29Z" message: InferenceService [error-404-isvc-74731] is Ready metadata: creationTimestamp: "2026-07-01T19:53:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:29Z" name: error-404-isvc-74731.18be427d81e6708c namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: eda8cca4-d10b-4e10-b6e5-2c716e258d1b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:01.472200Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fa17b5dd8 namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: 25f38623-8b23-4d6a-aa16-5caac63aa72e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fbbb9fdff namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: e538a26b-fce6-4e7b-ae99-a72a22a92759 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fbd3a06b1 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 72da361c-d551-4e5a-adbb-adc917ede29b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fc0067058 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 44e99028-e4d5-4977-a166-bd315a193cc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fc0693f96 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: d983fe5c-0c17-46a4-bbfb-f05f70ddbda3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fc075a63f namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 3bffb2c2-9d55-4c5b-a64c-f1e54bcec618 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fc4af52dd namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: ecdd658a-6560-4014-bd43-c3a19430ae81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be415fc513a871 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 5681dde4-8a32-4a39-8b2c-50de05932ac8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:33:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:40Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be41603ba84e8c namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: 26f7c5c0-ced0-4a18-b829-1a1c09f1c827 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be41e0d3f0bc90 namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: f27b1cea-f2f0-4b26-975c-4004197c751a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 69c2e01b-7582-4d6c-b35a-be506f1715cf kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s.18be41e0d3f115dd namespace: kserve-ci-e2e-test resourceVersion: "19616" uid: b0f77a95-e3df-41e9-af29-6cbc1d244c3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-834d0-predictor-5d6bdfcff8 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 9742014c-63db-4da0-93b9-f90295125820 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'Created pod: error-404-isvc-834d0-predictor-5d6bdfcff8-t9t2s' metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor-5d6bdfcff8.18be415fa0a5454b namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 4fc541a6-1eb9-412d-a43d-fd2609a6346d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: 29fe1cb8-d473-484f-9b97-49d31ebca884 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Scaled up replica set error-404-isvc-834d0-predictor-5d6bdfcff8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-834d0-predictor.18be415f9e24296a namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 7de71d5d-13ab-4d19-be72-19dae05a28e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: f9ad86a0-1f5e-4526-8d7a-2158f7aca293 kind: Event lastTimestamp: "2026-07-01T19:33:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:31Z" name: error-404-isvc-834d0-predictor.18be41631c7d6e1e namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 7b13ede2-63f1-4bed-804e-8d155001039d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: f9ad86a0-1f5e-4526-8d7a-2158f7aca293 kind: Event lastTimestamp: "2026-07-01T19:33:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:31Z" name: error-404-isvc-834d0-predictor.18be41631cfba31c namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 57ea2eb4-6834-4e51-9cdd-c24bcb9a3aa0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:33:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: f9ad86a0-1f5e-4526-8d7a-2158f7aca293 kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: error-404-isvc-834d0-predictor.18be416a1a4ecd93 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: d35b3a54-3b44-4fb1-bf3a-4d36fc04c37d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:33:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: f9ad86a0-1f5e-4526-8d7a-2158f7aca293 kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: error-404-isvc-834d0-predictor.18be416a1aca98d6 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: c79b57c4-5771-49ed-94f0-bb24d5f8d15e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 440a3d86-ee96-4053-b79f-f563a4d6f0e8 kind: Event lastTimestamp: "2026-07-01T19:33:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:50Z" name: error-404-isvc-834d0.18be415f9de9401d namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 01240c7c-624a-4e2c-8b16-941a73da5f2b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 440a3d86-ee96-4053-b79f-f563a4d6f0e8 kind: Event lastTimestamp: "2026-07-01T19:33:50Z" message: InferenceService [error-404-isvc-834d0] is Ready metadata: creationTimestamp: "2026-07-01T19:33:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:50Z" name: error-404-isvc-834d0.18be416af7526f97 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 8d776af6-5d69-4870-b9e6-3ab6d4213d6a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:44:26.337001Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-88843-predictor-5985fbc74-lkb8x to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff169aeb2b namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 17169f71-d162-49c0-b85f-33d65117ea37 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-88843-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff252a94b1 namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 7fd1904b-70f4-427d-aed2-192d0cc6aabe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff53f1e42c namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 3e9f7653-e323-4b41-8aa3-9eeeb9cf9cde reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff559c450e namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: d861c468-a8f9-4c5a-bab1-6e418d471318 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff586fbeae namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: a2e7c2de-4a4a-466c-9234-b5a15f944993 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff58cf8535 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: e190b177-354e-41ba-8953-6115461593fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff58d917ee namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 5fca9a20-1ebb-4280-ba8c-7bf001292f13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff5bab502e namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 9d05325c-0c33-4a0b-b055-2a4c46af5e04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:44:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:27Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff5c0f87d5 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 45a4f01a-dddb-462a-b78b-0f780c19d801 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:53:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:44Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be41ff7e54e360 namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: d8214d61-dfa8-4bee-928c-16cb0d2bce70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be428044262641 namespace: kserve-ci-e2e-test resourceVersion: "25429" uid: 6bda72b3-5be9-4fdb-989a-647c6bbcbab1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be42804427ad15 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: f575e6f3-6305-42e7-9b24-d63d2d4f2297 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88843-predictor-5985fbc74-lkb8x namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ef75e643-8bd8-4a66-b644-77f9ce967712 kind: Event lastTimestamp: "2026-07-01T19:53:44Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:44Z" name: error-404-isvc-88843-predictor-5985fbc74-lkb8x.18be4280f2c5bcce namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 312ed485-eeb7-4156-8bb7-f523c5a85290 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-88843-predictor-5985fbc74 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: bc85ba6e-e74e-4844-96db-b8a2669ffc40 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'Created pod: error-404-isvc-88843-predictor-5985fbc74-lkb8x' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-88843-predictor-5985fbc74.18be41ff1630ae1f namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 219156fc-abeb-49eb-b5f8-46ffec232657 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 499306b6-0dba-4786-90f8-5e198b2eea19 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Scaled up replica set error-404-isvc-88843-predictor-5985fbc74 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-88843-predictor.18be41ff147d3727 namespace: kserve-ci-e2e-test resourceVersion: "21175" uid: e07b43a8-1365-4f75-81d7-308efff5d5a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 2df54cee-cb46-4074-8e99-a2afcbd446ab kind: Event lastTimestamp: "2026-07-01T19:45:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:11Z" name: error-404-isvc-88843-predictor.18be4202931be53c namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 414c7eaf-93a4-4400-8849-607e62425ce6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 2df54cee-cb46-4074-8e99-a2afcbd446ab kind: Event lastTimestamp: "2026-07-01T19:45:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:11Z" name: error-404-isvc-88843-predictor.18be420293826b3d namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: a86cfeb0-a9a5-42ba-bb28-d7b64f4844a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 2df54cee-cb46-4074-8e99-a2afcbd446ab kind: Event lastTimestamp: "2026-07-01T19:45:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:41Z" name: error-404-isvc-88843-predictor.18be420d0fa8110b namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 465ea76e-c695-414d-961e-bff4e3215bce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 2df54cee-cb46-4074-8e99-a2afcbd446ab kind: Event lastTimestamp: "2026-07-01T19:45:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:41Z" name: error-404-isvc-88843-predictor.18be420d100e9869 namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: 762da862-82df-4ea1-b240-1695a1c2dba6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88843 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: c74cd73c-05d5-49a2-8525-78fcbe51f943 kind: Event lastTimestamp: "2026-07-01T19:45:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:45:14Z" name: error-404-isvc-88843.18be41ff14b12f84 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: 7dd253ab-b59e-44dc-ba35-412c84c32bb5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88843 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: c74cd73c-05d5-49a2-8525-78fcbe51f943 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-88843": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-88843": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-88843.18be41ff15208aa4 namespace: kserve-ci-e2e-test resourceVersion: "21179" uid: 261af982-26f0-48f1-8743-238a18bae711 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88843 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: c74cd73c-05d5-49a2-8525-78fcbe51f943 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-88843": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:26Z" name: error-404-isvc-88843.18be41ff1520e40e namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: ebda9ce0-5c2a-42c3-b0c0-76d1e07051d4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88843 namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: c74cd73c-05d5-49a2-8525-78fcbe51f943 kind: Event lastTimestamp: "2026-07-01T19:45:14Z" message: InferenceService [error-404-isvc-88843] is Ready metadata: creationTimestamp: "2026-07-01T19:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:45:14Z" name: error-404-isvc-88843.18be420a39c28525 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 3afa8b9f-9fac-4355-ac92-3ecc9c7bd6af reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:31.457045Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14187" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8fd9c-predictor-7658f9f556-xszkl to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:31Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414aac290609 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: bf15ebf8-692f-4a67-8512-29d952096ade reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:31Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414ac6cb3718 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: ee90f5dc-3201-4034-be06-2bc6e2a5ff28 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:31Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414ac9dba29b namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: 1bdba15b-c544-4d35-b28b-e65b251aa006 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" in 12.69s (12.69s including waiting). Image size: 1334624874 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414dbe4ac148 namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 36766f8b-1e13-4037-8920-8d2b37bb41e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414dc391a427 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 42097e37-dedf-4368-a7cd-78aafd80e0ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414dc4758857 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7463d912-91c0-4de8-9b36-5da4c8a0a176 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414dc484b41e namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: aaab9fbe-64ae-472b-81cb-5a77503a51d8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 8.075s (8.075s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:52Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414fa5d6c3d4 namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 6dc2b15e-67d4-451c-8083-afdcd0f5e44b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414fb0bd1fc1 namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 9c1b5110-ecd5-47a2-916b-b722371bf450 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414fb155a394 namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: b1a58b28-1b7c-48db-ae0a-ceb46fa78327 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:32:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:29Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be414fd926662c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: f4fd27d1-cc85-400e-8a04-da5dae93d11e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be415f9f7f6a9d namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 355561b3-8c2d-46cd-8454-e88720ca4029 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be415f9f7ff233 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 1fab3070-70d6-4bf6-88c8-ba55d972f964 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: b8ccbe83-6639-4eb3-9d3f-5dad47420d2a kind: Event lastTimestamp: "2026-07-01T19:33:04Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:04Z" name: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl.18be416061c9dbdc namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: ec1a37bc-327e-4930-b993-8aa5d5248d12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8fd9c-predictor-7658f9f556 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 788d109f-17d6-4ce8-b297-43317eba9513 kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: 'Created pod: error-404-isvc-8fd9c-predictor-7658f9f556-xszkl' metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: error-404-isvc-8fd9c-predictor-7658f9f556.18be414aab1ab218 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 622894a9-a0bb-4d91-8ac8-73d267061a88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 7c63c737-b4d3-42cd-a3cb-0aa20cf9b74d kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Scaled up replica set error-404-isvc-8fd9c-predictor-7658f9f556 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: error-404-isvc-8fd9c-predictor.18be414aa734bc0d namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: a960a971-8fd2-4fc5-9e0e-8df3535ebf76 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: f83affd5-fcc2-4293-86d6-f3fc2730c2cf kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:31Z" name: error-404-isvc-8fd9c-predictor.18be414e26ef210f namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 1240687c-d162-490e-b7a1-afd3bb839a4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: f83affd5-fcc2-4293-86d6-f3fc2730c2cf kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:31Z" name: error-404-isvc-8fd9c-predictor.18be414e276faf35 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 6c2a3ac1-6112-488b-a28f-a89bb27491c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: f83affd5-fcc2-4293-86d6-f3fc2730c2cf kind: Event lastTimestamp: "2026-07-01T19:32:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:46Z" name: error-404-isvc-8fd9c-predictor.18be415c23edc212 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 44fa17b6-f0ed-47f4-bc05-8aa8a8594cb6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: f83affd5-fcc2-4293-86d6-f3fc2730c2cf kind: Event lastTimestamp: "2026-07-01T19:32:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:46Z" name: error-404-isvc-8fd9c-predictor.18be415c2467357e namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 8e61fe9b-fe1d-4d99-83e9-b467c789da3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8fd9c namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: 002482fa-ab89-4795-a3cc-98cdbaa03d0d kind: Event lastTimestamp: "2026-07-01T19:32:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:39Z" name: error-404-isvc-8fd9c.18be414aa7bffc7d namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 67fa575d-3047-4dcb-bcef-0ad081900390 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8fd9c namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 002482fa-ab89-4795-a3cc-98cdbaa03d0d kind: Event lastTimestamp: "2026-07-01T19:32:39Z" message: InferenceService [error-404-isvc-8fd9c] is Ready metadata: creationTimestamp: "2026-07-01T19:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:39Z" name: error-404-isvc-8fd9c.18be415a9748f619 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 01dab8bf-3754-4e16-be4d-ea747b4a2f90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:16.314969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19590" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e0d0b024b5 namespace: kserve-ci-e2e-test resourceVersion: "19596" uid: 4747b44e-1c9c-40da-a20e-7a2e8fc1439a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ac147-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e0dd3995d2 namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: 6bc231c9-09b9-408d-9aad-efd91d4dc1ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e10eb115f2 namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: a2e1fe30-fc21-46b2-b820-0c1fdd7f7d2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e11018af33 namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 78e86eee-de8e-4b37-b3b1-12b2476b583d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e1135dcd25 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 6ad0af79-212e-4a99-bb83-7a3b81b20b22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e113d555ab namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: ee002978-74f9-48f1-b240-f48a407f493e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e113df3440 namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: 2ee056fd-735d-4c46-8554-079c474f8673 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e116f38cc9 namespace: kserve-ci-e2e-test resourceVersion: "19656" uid: 26e1574f-d206-4998-922d-d85afc4acc79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:17Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e117646965 namespace: kserve-ci-e2e-test resourceVersion: "19657" uid: 5531c811-9732-44d1-ae53-c6d32c515f6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:42:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:55Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41e18ec4e567 namespace: kserve-ci-e2e-test resourceVersion: "19951" uid: 1ff7f482-ceeb-40ce-9b6d-d9aad168a463 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41f13140ecf8 namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: a7891438-026f-49d7-84ba-245fb7c8c6e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 644393c3-9e8b-495f-8023-953890b7ef50 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8.18be41f131415659 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: bfef5baf-71e4-47a5-9c98-ea24d0148266 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ac147-predictor-84bcbc5868 namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 777063c2-8909-4289-8263-828296a91cb9 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Created pod: error-404-isvc-ac147-predictor-84bcbc5868-rqjk8' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-ac147-predictor-84bcbc5868.18be41e0d046bec7 namespace: kserve-ci-e2e-test resourceVersion: "19592" uid: 47e12ccd-73fb-4680-b53b-a76b84ebd09f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: 0a2d10c0-348c-405b-8e73-399e091c4b6f kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Scaled up replica set error-404-isvc-ac147-predictor-84bcbc5868 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-ac147-predictor.18be41e0cec38407 namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: c0b8ad5f-7737-4296-964f-659a302d455b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: a29f393f-3afd-4e49-8cbe-18a84d90c46a kind: Event lastTimestamp: "2026-07-01T19:42:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:46Z" name: error-404-isvc-ac147-predictor.18be41e44d369fdb namespace: kserve-ci-e2e-test resourceVersion: "19887" uid: a5d7ad07-6acf-4a37-b43c-185ea184b5f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: a29f393f-3afd-4e49-8cbe-18a84d90c46a kind: Event lastTimestamp: "2026-07-01T19:42:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:46Z" name: error-404-isvc-ac147-predictor.18be41e44d9ef209 namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: 3d685001-1422-4d70-a70c-11cb9c3ee9f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: a29f393f-3afd-4e49-8cbe-18a84d90c46a kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-ac147-predictor.18be41eb4afdb91f namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 7426e8b7-58c4-42a1-b321-27197797cae9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: a29f393f-3afd-4e49-8cbe-18a84d90c46a kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: error-404-isvc-ac147-predictor.18be41eb4b644e20 namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: dc9d578b-1dee-4136-8970-d83b3ae30eb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: b37446f5-17f0-498c-b3fb-750da77375b9 kind: Event lastTimestamp: "2026-07-01T19:43:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:05Z" name: error-404-isvc-ac147.18be41e0cecd1864 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 6ac86f0a-bc81-4307-bf4c-15dbe9e2736e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: b37446f5-17f0-498c-b3fb-750da77375b9 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-ac147": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ac147": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-ac147.18be41e0cf469cf7 namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 6c4063ed-6811-43f5-b8cc-dd9f34cda612 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: b37446f5-17f0-498c-b3fb-750da77375b9 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ac147": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:16Z" name: error-404-isvc-ac147.18be41e0cf46d9e3 namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: c17aac06-2418-4651-bd33-29840840bd60 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: b37446f5-17f0-498c-b3fb-750da77375b9 kind: Event lastTimestamp: "2026-07-01T19:43:05Z" message: InferenceService [error-404-isvc-ac147] is Ready metadata: creationTimestamp: "2026-07-01T19:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:05Z" name: error-404-isvc-ac147.18be41ec4a082d3e namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 508d106d-fa2b-4f2e-9a18-b6aaaddeb6df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:51.734979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26001" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282baf34fb3 namespace: kserve-ci-e2e-test resourceVersion: "26005" uid: 3573aa35-39f4-47d4-a4cb-325ea4988c45 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282d4d01c92 namespace: kserve-ci-e2e-test resourceVersion: "26064" uid: e9eba925-f8fa-43f6-aa69-afe33155f5e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282d63beb8a namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 20daaa07-8e95-412a-98ce-dd311f4be934 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282d96dbc72 namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 1b04e998-274d-4f14-9855-108579cea21e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282d9deb0f0 namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 046d6414-5957-4ea3-a3cf-3ea47702cdc0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282d9e7f71a namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: b2874b74-7ac0-4b5a-9b72-1c4295fe4f70 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282dcc493ef namespace: kserve-ci-e2e-test resourceVersion: "26072" uid: 9e7b4b9c-6546-4e0c-9c84-c517396b68ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4282dd396753 namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: d593f8ce-8ea8-4a5c-b3f8-5e658fd17a1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:53:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T20:03:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:09Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be428325a15fa5 namespace: kserve-ci-e2e-test resourceVersion: "30186" uid: 41357ae1-fd24-43d3-8ea5-2156d9723b26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T20:03:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:06Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4303e40a3811 namespace: kserve-ci-e2e-test resourceVersion: "30159" uid: 51641633-9025-498c-8d3c-864bfb7c5690 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T20:03:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:06Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be4303e40b5bad namespace: kserve-ci-e2e-test resourceVersion: "30160" uid: 0186d498-a8b3-4cae-a844-9f67cadc8915 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: 966a627b-3953-47a6-838b-afdc6b45ae43 kind: Event lastTimestamp: "2026-07-01T20:03:09Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:09Z" name: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj.18be43049a1fa74e namespace: kserve-ci-e2e-test resourceVersion: "30185" uid: 23f91080-df14-467b-9d17-e02d31519c62 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b4182-predictor-6877fd5bfd namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: fd1638a7-c8b0-4320-9e74-ff3983151dc3 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: 'Created pod: error-404-isvc-b4182-predictor-6877fd5bfd-bfpzj' metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-b4182-predictor-6877fd5bfd.18be4282ba79ae6f namespace: kserve-ci-e2e-test resourceVersion: "26002" uid: 54538275-fc4c-4498-939b-411de0e2ae83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: 7d605652-ecb6-4c29-a06c-8793adfa643c kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Scaled up replica set error-404-isvc-b4182-predictor-6877fd5bfd from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-b4182-predictor.18be4282b840e84e namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: b7d30d4b-586f-4176-8253-b227335df061 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:54:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: e3a0403c-66c3-4084-9d67-b4b6f634f813 kind: Event lastTimestamp: "2026-07-01T19:54:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:21Z" name: error-404-isvc-b4182-predictor.18be428636ae1e66 namespace: kserve-ci-e2e-test resourceVersion: "26349" uid: e386e07f-e8db-49cd-9542-6572cf1ee2c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:54:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: e3a0403c-66c3-4084-9d67-b4b6f634f813 kind: Event lastTimestamp: "2026-07-01T19:54:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:21Z" name: error-404-isvc-b4182-predictor.18be4286371154ff namespace: kserve-ci-e2e-test resourceVersion: "26350" uid: 5c1fa3ec-1776-478b-a719-2fa6883cc2e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:54:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: e3a0403c-66c3-4084-9d67-b4b6f634f813 kind: Event lastTimestamp: "2026-07-01T19:55:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:06Z" name: error-404-isvc-b4182-predictor.18be428d34b1bb2f namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f2061cde-6587-41e4-9754-7cddf94debb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:54:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: e3a0403c-66c3-4084-9d67-b4b6f634f813 kind: Event lastTimestamp: "2026-07-01T19:55:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:06Z" name: error-404-isvc-b4182-predictor.18be428d35278c11 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: cdea7c92-3199-485e-b9f2-d970946bed13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b4182 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: eda2ba93-e8d1-4a6b-aa6d-297b24cea02d kind: Event lastTimestamp: "2026-07-01T19:54:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:39Z" name: error-404-isvc-b4182.18be4282b83d04d0 namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 1b21ff44-a514-4022-b33e-19bf71d67a44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b4182 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: eda2ba93-e8d1-4a6b-aa6d-297b24cea02d kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-b4182": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b4182": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-b4182.18be4282c06f636a namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: d7185590-def2-4b9c-9890-ef936aac305d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b4182 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: eda2ba93-e8d1-4a6b-aa6d-297b24cea02d kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b4182": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:51Z" name: error-404-isvc-b4182.18be4282c06fab92 namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: f6ae106c-da4c-4255-9852-5a5af2bbb98d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b4182 namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: eda2ba93-e8d1-4a6b-aa6d-297b24cea02d kind: Event lastTimestamp: "2026-07-01T19:54:39Z" message: InferenceService [error-404-isvc-b4182] is Ready metadata: creationTimestamp: "2026-07-01T19:54:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:39Z" name: error-404-isvc-b4182.18be428de0ebbf3e namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: d483b457-39db-4b8c-92e8-afd49170d8f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:26.587894Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f12d484de6 namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: 512eb1e0-bce4-4372-ab28-da60747dedbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c6a40-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f136cb4241 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 31563710-e1e1-4a74-9ca9-c1b838dc2f2a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f16ab13c2a namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 770cc975-3c74-4da0-92db-87dd59e0474b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f16c176e76 namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: b99eed61-8f7b-4930-ae94-4dda8bb741f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f16fb29d18 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 3a138d19-f545-4753-b22f-a3cc286f0fad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f17014b223 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: d2cfab61-02ad-47a2-8c65-3124a3ee720f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f1701e754d namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 0ef9178e-c864-49c9-8477-5ab9a5785800 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f1733a2cbb namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: e835a5ce-7280-4091-b709-95bb0d5e7e9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f1739c604a namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: c8a678a4-771f-4070-81e8-d1600cef180c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:44:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:04Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be41f1b1f22ba7 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: ff760bfb-392b-4155-ba63-85eb2e1815aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be42725cab61ec namespace: kserve-ci-e2e-test resourceVersion: "24889" uid: 2b1d3dd1-1005-4b4f-b917-7e828f9dd65a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: c19349f6-8485-4781-b55e-3d6992cd8c96 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr.18be42725cabe97c namespace: kserve-ci-e2e-test resourceVersion: "24890" uid: 637cc5c6-698e-4ad2-a7c7-39c1f756a1f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c6a40-predictor-74cb7f48b4 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: f5f45e37-025c-40fa-990c-a8f239927fd6 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'Created pod: error-404-isvc-c6a40-predictor-74cb7f48b4-4tpdr' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-c6a40-predictor-74cb7f48b4.18be41f12c80b78e namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: 8a4f8dd0-deb0-4cff-8dc4-36f8c7fbb2ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 2d973353-c990-4f94-a0c9-e08cd7005288 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Scaled up replica set error-404-isvc-c6a40-predictor-74cb7f48b4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-c6a40-predictor.18be41f12adf0dba namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: cf3056d3-5869-4267-989e-29f50bf3e7f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 0bcfaca0-feab-4b00-bcb4-d39776a71c2f kind: Event lastTimestamp: "2026-07-01T19:44:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:11Z" name: error-404-isvc-c6a40-predictor.18be41f4a95a95a7 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 00cfae6b-f7ae-4b41-8885-b1c623a319e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 0bcfaca0-feab-4b00-bcb4-d39776a71c2f kind: Event lastTimestamp: "2026-07-01T19:44:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:11Z" name: error-404-isvc-c6a40-predictor.18be41f4a9a14671 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 5d35dbe6-7b30-438a-8533-929fac9b16af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 0bcfaca0-feab-4b00-bcb4-d39776a71c2f kind: Event lastTimestamp: "2026-07-01T19:44:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:41Z" name: error-404-isvc-c6a40-predictor.18be41ff25b35122 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: da48dada-0f07-4cbd-9af5-fafee5243dfe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 0bcfaca0-feab-4b00-bcb4-d39776a71c2f kind: Event lastTimestamp: "2026-07-01T19:44:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:41Z" name: error-404-isvc-c6a40-predictor.18be41ff26307fb1 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: f7242d62-40f7-4737-9b35-7b6e88cfd172 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: faf06bba-5aac-4538-92e3-cbf6bc33e140 kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:14Z" name: error-404-isvc-c6a40.18be41f12ae50073 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 8c8115d6-ce01-4115-8660-c3624d5ce415 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: faf06bba-5aac-4538-92e3-cbf6bc33e140 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-c6a40": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c6a40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-c6a40.18be41f12b63b066 namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: cec7770a-1edc-4375-9b8b-0d3599e981c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: faf06bba-5aac-4538-92e3-cbf6bc33e140 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c6a40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:26Z" name: error-404-isvc-c6a40.18be41f12b63f1c2 namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: b4bf9380-499e-4aea-8b76-511d5d72e6d0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: faf06bba-5aac-4538-92e3-cbf6bc33e140 kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: InferenceService [error-404-isvc-c6a40] is Ready metadata: creationTimestamp: "2026-07-01T19:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:14Z" name: error-404-isvc-c6a40.18be41fc6d75ab91 namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: 427a08bc-4a4e-4cb0-a88b-61336f4741c6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:41.326642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be42804e90b614 namespace: kserve-ci-e2e-test resourceVersion: "25507" uid: 1c539fa3-4a95-4e75-8a24-1c35b59d22f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25513" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be4280681e4405 namespace: kserve-ci-e2e-test resourceVersion: "25545" uid: 9538b253-73f3-46a4-9b66-bf4785f1c91b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be428069c65f4c namespace: kserve-ci-e2e-test resourceVersion: "25548" uid: e8fd1eb4-09d3-4142-88f3-8d2b8784f56f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be42806ca8bfd5 namespace: kserve-ci-e2e-test resourceVersion: "25550" uid: 2d2d73e1-c665-493b-a221-9ca3f814fcaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be42806d1b13c9 namespace: kserve-ci-e2e-test resourceVersion: "25551" uid: 781d2c01-8a18-4d1c-a06d-f575c8ab0c1f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be42806d25006d namespace: kserve-ci-e2e-test resourceVersion: "25552" uid: 0c2b302b-ca30-471b-b4af-0601c5709600 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be428070afe9f1 namespace: kserve-ci-e2e-test resourceVersion: "25553" uid: 0eb0c26d-29c8-4911-bebd-d78a620d6b43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be4280712470e9 namespace: kserve-ci-e2e-test resourceVersion: "25554" uid: 328da4cb-e53d-4745-b8ea-5e393f2b6f04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T19:54:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:19Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be4280ce8ee114 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: 4091891e-cc03-4252-9a51-77bf27d97beb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T20:11:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:00Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be4372572a57ec namespace: kserve-ci-e2e-test resourceVersion: "33447" uid: a0496b5a-cbf3-4b88-9349-3915eda4fa6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: c65dc835-3da6-49c1-b1da-623bc534f522 kind: Event lastTimestamp: "2026-07-01T20:11:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:00Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888.18be437257347033 namespace: kserve-ci-e2e-test resourceVersion: "33448" uid: e8ce7441-9fec-492d-8a44-a58aa9aec2c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ee34f-predictor-6c5b4cbc46 namespace: kserve-ci-e2e-test resourceVersion: "25480" uid: 3ca17e33-eb4c-4bdf-aadd-bc8d7a01db3b kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created pod: error-404-isvc-ee34f-predictor-6c5b4cbc46-zh888' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor-6c5b4cbc46.18be42804e029c01 namespace: kserve-ci-e2e-test resourceVersion: "25500" uid: b4a5043c-bd8a-4177-8e88-d09d08ecf641 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25473" uid: 6fc76406-5447-43dd-aa10-9b06305ddd9c kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Scaled up replica set error-404-isvc-ee34f-predictor-6c5b4cbc46 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f-predictor.18be42804c03319a namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: 2e7231b6-1529-4eb4-ad96-3b330180d8fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 30800512-3597-47d4-b4e0-793295800336 kind: Event lastTimestamp: "2026-07-01T19:54:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:26Z" name: error-404-isvc-ee34f-predictor.18be4283cab35be8 namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: 11913691-d1a1-41cc-a4f4-ecda7b59ebde reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 30800512-3597-47d4-b4e0-793295800336 kind: Event lastTimestamp: "2026-07-01T19:54:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:26Z" name: error-404-isvc-ee34f-predictor.18be4283cb1abf3d namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: 471fc29d-6c93-49cb-a60f-6349cb1df7af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: 30800512-3597-47d4-b4e0-793295800336 kind: Event lastTimestamp: "2026-07-01T19:55:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:11Z" name: error-404-isvc-ee34f-predictor.18be428e48dcc858 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: 035aee74-5dea-4dc0-9fb9-dc7f1cb2afb4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: 30800512-3597-47d4-b4e0-793295800336 kind: Event lastTimestamp: "2026-07-01T19:55:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:11Z" name: error-404-isvc-ee34f-predictor.18be428e495bc895 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 2e8dbe80-6fce-47c1-8871-3d4fe63836de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ee34f namespace: kserve-ci-e2e-test resourceVersion: "25447" uid: c3a8ff8b-238f-45d4-a1a5-8cd42982a445 kind: Event lastTimestamp: "2026-07-01T19:54:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:29Z" name: error-404-isvc-ee34f.18be42804c7ba20a namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: d22a9855-1019-44d2-85ce-6e14cf4df2b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ee34f namespace: kserve-ci-e2e-test resourceVersion: "25447" uid: c3a8ff8b-238f-45d4-a1a5-8cd42982a445 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-ee34f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ee34f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f.18be42804db670e3 namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: c4f8ac12-3ead-484f-b07a-acadff7ce2b1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ee34f namespace: kserve-ci-e2e-test resourceVersion: "25447" uid: c3a8ff8b-238f-45d4-a1a5-8cd42982a445 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ee34f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:41Z" name: error-404-isvc-ee34f.18be42804db6bc83 namespace: kserve-ci-e2e-test resourceVersion: "25506" uid: fe3b98b2-d02b-4a44-8f37-da4b9af03009 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: c3a8ff8b-238f-45d4-a1a5-8cd42982a445 kind: Event lastTimestamp: "2026-07-01T19:54:29Z" message: InferenceService [error-404-isvc-ee34f] is Ready metadata: creationTimestamp: "2026-07-01T19:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:29Z" name: error-404-isvc-ee34f.18be428b8a721fb6 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 99e274d4-537d-4fa7-9a64-8ef43ea15b97 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:31.269449Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-875bcd459-8dd72 to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414aa0fa980f namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6bed3c58-4acf-4b46-a41c-057a62b72739 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414abad4d56d namespace: kserve-ci-e2e-test resourceVersion: "14214" uid: 4ece4147-ce86-4db5-b30e-441aff10fe99 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:359d904aaa602365d470b7fafd3828b888fb4c1e71dc1473bbad336c728c7088" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414abdc530ab namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 42d94ee0-739b-4c7b-a205-6ca52f144536 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414ac204db19 namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 33202c97-6a23-40b2-a8e8-bfb952c1a9d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414ac28d9e24 namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 14ec1437-3f23-4d76-93fd-e93e72c909c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T19:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:37Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414c18e3f5c8 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: e6c394ce-209b-492b-8115-4f528f560590 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:52Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 15.348s (15.348s including waiting). Image size: 1558595510 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:52Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414fabb9083e namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: b41d1e53-342f-4f45-bbfd-bb58fa39c707 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414fb164177a namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 9d31be1e-0667-478b-a587-ebf539184379 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414fb1f9f909 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 797be11b-21b1-447c-944a-23856879df01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414fb2135169 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: fc17a803-c747-496e-8077-78e04dbe5892 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414fb692d400 namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: dcc6bfd1-fe14-46c6-8988-664dc233ba75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be414fb7135e48 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 7993a7de-ddc7-4b4d-ad48-09bdedeee73e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T19:31:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:33:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:18Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be4150158a8a20 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: d4dcbd27-fca0-4709-b732-187908d2639b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be416d880e2d16 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: 53a0ca65-97a2-4345-a101-ed08566ebb78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be416d880f4634 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: b0f00023-bfd9-45ea-8fd1-f7eef3df2425 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 4843b39d-a468-4c30-863a-72175b1ab0fb kind: Event lastTimestamp: "2026-07-01T19:34:05Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:34:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:05Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8dd72.18be416e95fea4af namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: f3025789-31bb-46dc-9b98-8e9da617aef5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-875bcd459 namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: dedace60-b54b-4e82-8964-2d305348340f kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-875bcd459-8dd72' metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor-875bcd459.18be414a9faa04e5 namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: e1d7e323-22fc-455c-84b8-4ff8471aa07c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: e7db8e1a-2097-419e-9fca-a1aac55a7528 kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-875bcd459 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-1-predictor.18be414a9a000604 namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: 936f4044-7457-48e5-92e2-1d3b6c8de9aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 04ac7458-1402-4f14-939f-2a8148987d60 kind: Event lastTimestamp: "2026-07-01T19:32:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:16Z" name: isvc-sklearn-graph-1-predictor.18be414e19eeafc0 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 823f41ae-18e0-4f66-86a3-3a925b58a87a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 04ac7458-1402-4f14-939f-2a8148987d60 kind: Event lastTimestamp: "2026-07-01T19:32:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:16Z" name: isvc-sklearn-graph-1-predictor.18be414e1b41bd8c namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: e3287ad7-57ac-440a-94d8-a98ac6d62d23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:32:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 04ac7458-1402-4f14-939f-2a8148987d60 kind: Event lastTimestamp: "2026-07-01T19:33:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:46Z" name: isvc-sklearn-graph-1-predictor.18be415898a38af9 namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: 03e226a3-feed-4836-aaf4-a25bd75e8685 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:32:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 04ac7458-1402-4f14-939f-2a8148987d60 kind: Event lastTimestamp: "2026-07-01T19:33:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:46Z" name: isvc-sklearn-graph-1-predictor.18be415899678b2f namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 9e71abf9-48e6-45c7-b62e-f3601a191d51 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14122" uid: 09b44ab1-4806-457a-a9c4-bda6df5d5894 kind: Event lastTimestamp: "2026-07-01T19:33:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:28Z" name: isvc-sklearn-graph-1.18be414a9a475a00 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: bc9b8c2b-d353-4985-ae25-08557cf56902 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 09b44ab1-4806-457a-a9c4-bda6df5d5894 kind: Event lastTimestamp: "2026-07-01T19:33:28Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T19:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:28Z" name: isvc-sklearn-graph-1.18be4165e4378ec7 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: 6392aeb5-94a7-4be2-8965-615a043c2156 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:32.089750Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:32Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414ad1df5e3c namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 9f258a49-2dde-427f-bd20-e8a4dbb2a109 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:33Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:33Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414b13728a00 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: fe83b89d-f8cd-4e97-ae16-f61e1c87c9a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:359d904aaa602365d470b7fafd3828b888fb4c1e71dc1473bbad336c728c7088" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:33Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414b1662095d namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: c3dc32c0-76d0-48e1-99f5-21c4abb6cc27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T19:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:33Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414b1a95efdc namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 0a22ca16-ddc3-4a77-aa4b-2262386fbb65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T19:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:33Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414b1c9697f9 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: 27c406da-7632-4637-b2d2-6268dedd4684 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:37Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T19:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:37Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414c18b75472 namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: 4de53cc9-1df7-40a2-9030-1bd84d9d092b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:52Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 15.302s (15.302s including waiting). Image size: 1558595510 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:52Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fa8cc5c6f namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 2fc9bc4d-2134-4441-b28c-6f36666d7726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fb1c23ccc namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: e0091b22-b396-45fe-a598-308d7077a3a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fb266ef65 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: a50867f3-cbf9-43b0-acb7-7c61425f07a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fb280ddc3 namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: b9ff5737-222a-4d31-aff0-f32b0cd66218 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fb6b68fbb namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 4df4ed56-76c3-4778-9171-c405276d5e66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:31:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:53Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fb72e7741 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 4ac62acc-178f-4c09-bcbc-6dfc7b0d52ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T19:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:34:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:04Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be414fd94de3c0 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: be1921e6-2dd9-4ebf-9df7-cc26f9465e38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be416d826d5654 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 4876bc5a-8e67-462d-89af-6c1bc4162f95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be416d826e087f namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: f4b9dde1-f02b-4d46-8a0e-59f18a8b98d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 1f6bb43c-c710-47e2-9f14-5f14ef6eac68 kind: Event lastTimestamp: "2026-07-01T19:34:04Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:04Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c.18be416e5a2786b8 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 97b231a8-f4d9-451b-9a67-333e8935738e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6c8478f9b5 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 3a35f181-afdd-45bf-b4b9-7f558cbba98c kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6c8478f9b5-rzr4c' metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:32Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5.18be414acfd27a5f namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 6e215d9b-0967-4dfc-a186-e76e44ea4262 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: a9dea662-7e49-444f-9fc1-66d54a2a6173 kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6c8478f9b5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: isvc-sklearn-graph-2-predictor.18be414ac4b5c534 namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 41810101-6e8a-4359-a76d-6ce67b103964 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: a28032af-9815-4885-802f-9cd1cd5200b8 kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:31Z" name: isvc-sklearn-graph-2-predictor.18be414e441e375b namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: e906f46f-0fa9-4b96-83ff-1ec2d7e010c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: a28032af-9815-4885-802f-9cd1cd5200b8 kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:31Z" name: isvc-sklearn-graph-2-predictor.18be414e44ba62d7 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: c1df31b5-622d-4497-b17d-0ed5fcea9f32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: a28032af-9815-4885-802f-9cd1cd5200b8 kind: Event lastTimestamp: "2026-07-01T19:33:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:47Z" name: isvc-sklearn-graph-2-predictor.18be415c4064ce2b namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: c439e498-cbcf-46bc-ad02-8c7e4269a859 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: a28032af-9815-4885-802f-9cd1cd5200b8 kind: Event lastTimestamp: "2026-07-01T19:33:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:47Z" name: isvc-sklearn-graph-2-predictor.18be415c4120cb76 namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: cbdfb3a2-8fb9-4190-bbfd-a176fd280d62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 5dd1878f-c0b1-476c-afd7-fdd8c173a0ac kind: Event lastTimestamp: "2026-07-01T19:33:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:24Z" name: isvc-sklearn-graph-2.18be414ac4bc5de9 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: c7dfb130-d481-417f-ae08-e674135a0b3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 5dd1878f-c0b1-476c-afd7-fdd8c173a0ac kind: Event lastTimestamp: "2026-07-01T19:33:24Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T19:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:24Z" name: isvc-sklearn-graph-2.18be4164fab5f986 namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: 860e6c0c-c8dc-4ae3-810b-91c5b356b1f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:31.767610Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-7ll6v to ip-10-0-134-212.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be414abeabe83b namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 438a5a4b-f21b-4cb0-9446-a47a87db4ac5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be414adab33031 namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 63d56611-a0fd-4bdc-ae44-6070ee68c4be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:359d904aaa602365d470b7fafd3828b888fb4c1e71dc1473bbad336c728c7088" already present on machine metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be414adce48b9d namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 1fcd5c37-246e-4527-a056-822bef2c2d6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be414ae10d6629 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 6049a972-f601-4b49-904f-f94fc7b64f7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be414ae1a6b3e3 namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: 8b6864a9-d3f9-43e4-841d-01dcd5f179ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:31:37Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T19:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be414c18d2b76a namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: c005e436-4c42-410e-b5db-f7a23e6c18ba reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:32:06Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.903s (28.903s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T19:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be4152d3a1bd94 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 2eb4c167-6501-4bd7-a7a9-0fbd263d307e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:32:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be4152da97dd87 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 941cf6b0-577e-4438-a845-6f104cec3627 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:32:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be4152db470eaf namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 4679956d-0404-4ea8-8bd1-914dc04e1d02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:32:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be4152db5ca889 namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 83f8f920-1129-44e0-916a-212136267f06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:32:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be4152e133867b namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 248a518b-06ce-4679-89ba-9b1ef9e7284a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:32:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be4152e2674ded namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: fa338760-445e-490b-940d-d60662119d18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T19:32:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:34:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be41532096107a namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 51f33bbc-e889-4dbf-b1e2-5a46043ab4e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be416d7dc47bc3 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 279d956b-8cf4-408a-96d9-06c17d91358d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be416d7dc5088d namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 32dbb211-a1c5-45f9-9aa5-7b7529989ce3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 317d360c-a0c9-47fb-8fc3-c5b4622bda0b kind: Event lastTimestamp: "2026-07-01T19:34:03Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-7ll6v.18be416e23081952 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: daddf8e4-1c47-47fb-a58f-40fd785a271f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-212.ec2.internal source: component: kubelet host: ip-10-0-134-212.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: baafbb41-ec81-4a31-8102-41257edfd83b kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-7ll6v' metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be414abd7d5db1 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 9fcbf62e-ef84-430e-a81e-a129be9cc5a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: a7537f6d-b900-4173-9f48-c30ddf6b657a kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:31Z" name: isvc-xgboost-graph-predictor.18be414ab4e35a8b namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: 5fc76291-693d-4a57-807a-97ed6f2b962e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: bdaf97b0-1edc-4160-89d9-f4cef3962f25 kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:31Z" name: isvc-xgboost-graph-predictor.18be414e33bf2985 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 179eb07a-a99f-48c3-b833-55f78838cb87 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: bdaf97b0-1edc-4160-89d9-f4cef3962f25 kind: Event lastTimestamp: "2026-07-01T19:32:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:31Z" name: isvc-xgboost-graph-predictor.18be414e347215d2 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: c2b76a19-a45d-4fa9-8418-88de00926318 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: bdaf97b0-1edc-4160-89d9-f4cef3962f25 kind: Event lastTimestamp: "2026-07-01T19:33:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:31Z" name: isvc-xgboost-graph-predictor.18be415c314289e6 namespace: kserve-ci-e2e-test resourceVersion: "15557" uid: 3660f5c5-dae0-4621-afa7-11964da500f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: bdaf97b0-1edc-4160-89d9-f4cef3962f25 kind: Event lastTimestamp: "2026-07-01T19:33:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:31Z" name: isvc-xgboost-graph-predictor.18be415c31d5d1e9 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: 859d452f-81ce-4fbe-a452-31641259b156 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: f7d1c3be-5be4-43b9-9e7e-58d4d02473e1 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:13Z" name: isvc-xgboost-graph.18be414ab46e2417 namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: ca31c672-7d34-4ed0-a977-c238fd85d4be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: f7d1c3be-5be4-43b9-9e7e-58d4d02473e1 kind: Event lastTimestamp: "2026-07-01T19:33:13Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T19:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:13Z" name: isvc-xgboost-graph.18be41628543ba13 namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: ce736247-4afa-441c-a050-1758f5aea7b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:50.807804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-549d845845-lqf56 to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:50Z" name: model-chainer-549d845845-lqf56.18be416b1e1cd90c namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 7ff79e25-76e1-4e80-a6a1-15c5f457df2d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:50Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:33:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:33:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:50Z" name: model-chainer-549d845845-lqf56.18be416b25458ec1 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: c8fd5d33-b364-4463-b9fb-ccaaaba9f182 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:51Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:33:51Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:51Z" name: model-chainer-549d845845-lqf56.18be416b5b76ea02 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 107cd7d2-237f-4197-b573-357f4b76873c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:33:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:51Z" name: model-chainer-549d845845-lqf56.18be416b5cbf0bf5 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: b2215d19-2691-4c6b-a2c2-ed111f942d97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:33:51Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T19:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:51Z" name: model-chainer-549d845845-lqf56.18be416b60e75db8 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 85b65d85-faef-44c7-97bd-9671f5978e9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:33:51Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T19:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:51Z" name: model-chainer-549d845845-lqf56.18be416b615bbd9c namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 9dbc023f-1095-47e8-947d-bd604447f627 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:34:00Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T19:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:00Z" name: model-chainer-549d845845-lqf56.18be416d77f26014 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: d8e8f2ab-7155-4f15-9ea5-4abf18b558de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-549d845845-lqf56 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: ced921cf-2140-4212-b411-d494b9ca0d41 kind: Event lastTimestamp: "2026-07-01T19:34:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:28Z" name: model-chainer-549d845845-lqf56.18be416e229af994 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 56a01701-e998-45fa-aead-0c26f423bc8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-549d845845 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 68b1f1e4-515e-4590-b660-a30001e5baed kind: Event lastTimestamp: "2026-07-01T19:33:50Z" message: 'Created pod: model-chainer-549d845845-lqf56' metadata: creationTimestamp: "2026-07-01T19:33:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:50Z" name: model-chainer-549d845845.18be416b1d9fe4f1 namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 5d3b4991-34f6-4fe9-a0bf-7a6795103e8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 1bcf6887-82ab-4342-8821-8172699b6070 kind: Event lastTimestamp: "2026-07-01T19:33:50Z" message: Scaled up replica set model-chainer-549d845845 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:50Z" name: model-chainer.18be416b1cb45375 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: a5a61f6f-a0fb-4fa6-ae1d-b218c3a01e35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 2eff5753-6df3-4964-aa7b-37111a4c1d5f kind: Event lastTimestamp: "2026-07-01T19:33:50Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T19:33:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:50Z" name: model-chainer.18be416b22bd1795 namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 87f9a8cb-f0a9-4c80-967d-929334e12d53 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:35:01.100079Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-01ba2-68958d669b-8zsrs to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:35:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:35:01Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be417b7bdbf4b3 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 39699c47-6f62-4bac-b5f6-3fc1d4243c93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:35:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-01ba2-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:35:01Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be417b8746a4ee namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: d34be0ba-0a7f-4552-b857-c79958440528 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:35:02Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:35:02Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be417bb96d9b65 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 0680cf2d-cb63-4be5-8bb6-5b190486d303 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-01ba2} kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:35:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:35:02Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be417bbaa8bf55 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 196595fd-2477-4107-a6c6-94db0180bf36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-01ba2} kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:35:02Z" message: 'Created container: sequence-graph-01ba2' metadata: creationTimestamp: "2026-07-01T19:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:35:02Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be417bbdb51863 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: 8143ee38-2cd7-4f91-8457-ef75479244d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-01ba2} kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:35:02Z" message: Started container sequence-graph-01ba2 metadata: creationTimestamp: "2026-07-01T19:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:35:02Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be417bbe335e2e namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 31ac65bf-ba68-4383-8efd-9895693a0d2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-01ba2} kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: Stopping container sequence-graph-01ba2 metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:15Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be41eeadca1a40 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: aa55ec31-0489-4533-9c45-c981a30abb36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-01ba2} kind: Pod name: sequence-graph-01ba2-68958d669b-8zsrs namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 75fe986e-063f-4101-8dee-3b49f2451d6a kind: Event lastTimestamp: "2026-07-01T19:43:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:44Z" name: sequence-graph-01ba2-68958d669b-8zsrs.18be41ef6dbdcf14 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 00a02de5-08f1-4a66-ab31-3c79ab5399fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-01ba2-68958d669b namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: c840f02c-dfe4-412d-b51b-5e8a3f8a1fca kind: Event lastTimestamp: "2026-07-01T19:35:01Z" message: 'Created pod: sequence-graph-01ba2-68958d669b-8zsrs' metadata: creationTimestamp: "2026-07-01T19:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:01Z" name: sequence-graph-01ba2-68958d669b.18be417b7b632db3 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: aaacafd4-4432-415d-a1ca-53084ea766a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: b29dc183-261f-424a-a15b-4d3a3d946afd kind: Event lastTimestamp: "2026-07-01T19:35:01Z" message: Scaled up replica set sequence-graph-01ba2-68958d669b from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:01Z" name: sequence-graph-01ba2.18be417b7a73a517 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 43f4ba18-eb92-45b0-b0cb-0d3aa855ac07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:35:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 994588d2-63ef-4087-bf4e-7058a1c2ed6c kind: Event lastTimestamp: "2026-07-01T19:35:01Z" message: InferenceGraph [sequence-graph-01ba2] is Ready metadata: creationTimestamp: "2026-07-01T19:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:35:01Z" name: sequence-graph-01ba2.18be417b878d9fdf namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 8c1c436f-d0bf-473c-b3cb-0e317632b104 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:35:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 29211ae1-2665-4049-bf26-7a72275c818e kind: Event lastTimestamp: "2026-07-01T19:35:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:35:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:46Z" name: sequence-graph-01ba2.18be417ef8fc87fa namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: fccf2da6-1a8c-47a9-bef1-ba169c3d19ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:35:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 29211ae1-2665-4049-bf26-7a72275c818e kind: Event lastTimestamp: "2026-07-01T19:35:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:35:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:46Z" name: sequence-graph-01ba2.18be417ef966be2c namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: 15434e7c-3d46-4a53-8a44-e7a305606ba8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:44:16.028993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-46ddd-6684c676db-rs8pj to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41fcb0331dfd namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 838b18bf-f190-4df0-8a1c-b188625adade reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41fcc9be6a8a namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 62a287b0-03a4-4fd9-96d7-53dc63adae06 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-46ddd} kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41fccb07f4e0 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 6ae755f7-8468-4fc4-b87a-046b7eaa9681 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-46ddd} kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: 'Created container: sequence-graph-46ddd' metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41fcce05ce4a namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 880d8a7f-0f18-4a45-87d0-a475aa5a9fbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-46ddd} kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: Started container sequence-graph-46ddd metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41fcce80f50c namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 160e5ad0-70da-4891-85fb-4b633736c6e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-46ddd} kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Stopping container sequence-graph-46ddd metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41ff08a3c448 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 9cf2abfa-18a4-40d6-9854-8e80168162f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-46ddd} kind: Pod name: sequence-graph-46ddd-6684c676db-rs8pj namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: e0f27703-d667-45ea-8361-c30224333bff kind: Event lastTimestamp: "2026-07-01T19:44:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:53Z" name: sequence-graph-46ddd-6684c676db-rs8pj.18be41ff999f2a99 namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: 4a5b04f4-3201-4711-9650-08565d13278b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-46ddd-6684c676db namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: d642c431-1837-4fda-b390-80c57d8b0876 kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: 'Created pod: sequence-graph-46ddd-6684c676db-rs8pj' metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd-6684c676db.18be41fcaf7663ac namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 7f5f0dff-cc80-4492-956a-64521eb649f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: ea83b4db-07ba-46e7-88ac-0e00de06f126 kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: Scaled up replica set sequence-graph-46ddd-6684c676db from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd.18be41fcaea19e7f namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: d9654121-a3e8-41c7-926b-240bbb915dfe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 6d97191d-5181-408c-b0e3-1445abbb9a85 kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: InferenceGraph [sequence-graph-46ddd] is Ready metadata: creationTimestamp: "2026-07-01T19:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:16Z" name: sequence-graph-46ddd.18be41fcb2b1ab83 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: f949636f-c267-4f3a-a1af-ac41a59eb952 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:45:26.305984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-88843-5799f5fbfd-cbgsd to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be420d0d08ee76 namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: 273d1673-0291-42cd-a97d-802bec250555 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be420d26ba7ca5 namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 10293733-a37a-40f5-9a63-0d53197b6c50 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88843} kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be420d27ff9399 namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 04579560-bf2b-446a-a649-50d686b6d6dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88843} kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: 'Created container: sequence-graph-88843' metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be420d2abc0197 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 5a293768-a028-4637-bc3e-b0e87011743b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88843} kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: Started container sequence-graph-88843 metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be420d2b49bc83 namespace: kserve-ci-e2e-test resourceVersion: "21802" uid: fb8a117e-adcf-4171-8672-3f48229258fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88843} kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Stopping container sequence-graph-88843 metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be42803d75523d namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: fce67a44-1844-4207-bf54-ea7ce8445af7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88843} kind: Pod name: sequence-graph-88843-5799f5fbfd-cbgsd namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 6e81ab7d-29e1-46b0-997c-542f6571299c kind: Event lastTimestamp: "2026-07-01T19:54:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:07Z" name: sequence-graph-88843-5799f5fbfd-cbgsd.18be4280a9427344 namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: f228fe4b-62dd-429b-b9e1-a7191133864a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-88843-5799f5fbfd namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: 931cca38-3247-4696-b945-9725588610c1 kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: 'Created pod: sequence-graph-88843-5799f5fbfd-cbgsd' metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843-5799f5fbfd.18be420d0cb0e988 namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: d23c8b8d-314c-4fcb-b37c-e15ab9561bcc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-88843 namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: d49bd3de-f7cf-40af-9718-76188fea04fe kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: Scaled up replica set sequence-graph-88843-5799f5fbfd from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843.18be420d0bd2f9bd namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 905415b4-46cd-4968-9f22-0d1f650f6792 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-88843 namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 6b35b615-0930-496d-99bb-68117ed78304 kind: Event lastTimestamp: "2026-07-01T19:45:26Z" message: InferenceGraph [sequence-graph-88843] is Ready metadata: creationTimestamp: "2026-07-01T19:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:45:26Z" name: sequence-graph-88843.18be420d11e703cf namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 65f88b66-7899-4ae3-bb10-4de967edd106 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:45:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-88843 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 2273efc7-5529-4864-bb2b-1fd21e939402 kind: Event lastTimestamp: "2026-07-01T19:46:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:46:11Z" name: sequence-graph-88843.18be42108a482c6d namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: 982b7cca-b8aa-4444-8e0a-e8e92a85fc4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:45:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-88843 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 2273efc7-5529-4864-bb2b-1fd21e939402 kind: Event lastTimestamp: "2026-07-01T19:46:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:45:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:46:11Z" name: sequence-graph-88843.18be42108ab75e5f namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: 86e9745c-182a-4c6f-8e61-0f21a5e34068 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:41.317223Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-74731-594747d957-djfc8 to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731-594747d957-djfc8.18be42804e010ea7 namespace: kserve-ci-e2e-test resourceVersion: "25505" uid: b74670a9-c276-4b0d-b8df-9e3ebabd1c52 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-74731-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731-594747d957-djfc8.18be42805fb3530e namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: eb77ec02-4053-432d-a373-6b2e0c7f85df reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25511" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:53:42Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:42Z" name: splitter-graph-74731-594747d957-djfc8.18be42808b55f632 namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: 19e6e5f8-77ae-4993-aafd-2eef06579f8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-74731} kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:53:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:42Z" name: splitter-graph-74731-594747d957-djfc8.18be42808c9d7be4 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: c41021c8-2934-4a3a-bdae-e86dd2447b04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-74731} kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:53:42Z" message: 'Created container: splitter-graph-74731' metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:42Z" name: splitter-graph-74731-594747d957-djfc8.18be428090142fa9 namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 1845fe1a-32fc-4ef7-8625-e1a41da9ccc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-74731} kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:53:42Z" message: Started container splitter-graph-74731 metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:42Z" name: splitter-graph-74731-594747d957-djfc8.18be428090e20485 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: fb632aee-00b9-4d6a-b248-a775d09c6bc9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-74731} kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Stopping container splitter-graph-74731 metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:51Z" name: splitter-graph-74731-594747d957-djfc8.18be4282a5c1ef98 namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: c6fcc0c2-aefe-4172-8745-c20c3c28ab84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-74731} kind: Pod name: splitter-graph-74731-594747d957-djfc8 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: fe83d95f-4730-40b4-be32-ac9052550795 kind: Event lastTimestamp: "2026-07-01T19:54:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:19Z" name: splitter-graph-74731-594747d957-djfc8.18be4283445f902d namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 36bf77b8-8879-4831-a327-8c63cfe6d6ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-74731-594747d957 namespace: kserve-ci-e2e-test resourceVersion: "25487" uid: 7b10af50-790c-45d5-93c1-4f9a29aa907c kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created pod: splitter-graph-74731-594747d957-djfc8' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731-594747d957.18be42804d6f98a1 namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 03f5896a-4e3a-4e0c-a270-293e65a8e3d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-74731 namespace: kserve-ci-e2e-test resourceVersion: "25478" uid: 9c3dc353-eabb-423b-a915-8faa60d48de8 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Scaled up replica set splitter-graph-74731-594747d957 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731.18be42804cbae1ca namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 8cd5c111-3b3e-4163-a3a8-e3fc35ffe44e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-74731 namespace: kserve-ci-e2e-test resourceVersion: "25519" uid: d5acec21-a609-4be3-800b-f68092ced998 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: InferenceGraph [splitter-graph-74731] is Ready metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731.18be428053949aa2 namespace: kserve-ci-e2e-test resourceVersion: "25521" uid: 6809c049-2910-4c8c-b259-074fde0227df reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-74731 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: d5acec21-a609-4be3-800b-f68092ced998 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Failed to update status for InferenceGraph "splitter-graph-74731": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-74731": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731.18be4280560008de namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: 1002f009-d01c-4137-a8bb-c7d03830f6a1 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-74731 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: d5acec21-a609-4be3-800b-f68092ced998 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-74731": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:41Z" name: splitter-graph-74731.18be42805600498c namespace: kserve-ci-e2e-test resourceVersion: "25527" uid: 227c3d12-f085-41b6-9ec9-e2579e558114 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:54:51.581089Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b4182-58cdcd74d5-w67h4 to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:54:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:54:51Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4290aa0e95b0 namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: ed2ff2b9-b57c-42a2-8c4e-6f646d289658 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T19:54:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-b4182-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:51Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4290b08b2eb8 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: f15c7657-8433-43c8-abb1-a21fa6ab42da reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T19:54:52Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:54:52Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4290d5a51fd9 namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: 456f361f-1773-4e7c-a528-d134daeadaad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b4182} kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T19:54:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:52Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4290d727b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: 31fbea99-3a8c-4ae2-b7bc-30b45513ac5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b4182} kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T19:54:52Z" message: 'Created container: splitter-graph-b4182' metadata: creationTimestamp: "2026-07-01T19:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:52Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4290da7e4013 namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 9e669a34-88e7-4c4d-8a81-176220c91519 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b4182} kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T19:54:52Z" message: Started container splitter-graph-b4182 metadata: creationTimestamp: "2026-07-01T19:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:52Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4290daf3e7ef namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 45863aca-dd11-4556-b402-7979936098b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b4182} kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T20:03:06Z" message: Stopping container splitter-graph-b4182 metadata: creationTimestamp: "2026-07-01T20:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:06Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be4303d95ef8c5 namespace: kserve-ci-e2e-test resourceVersion: "30128" uid: e8356831-cf36-48e8-b28a-fa62e6bc542f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b4182} kind: Pod name: splitter-graph-b4182-58cdcd74d5-w67h4 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 90053a3e-e1d7-4367-9bcf-44f26a7c0ce8 kind: Event lastTimestamp: "2026-07-01T20:03:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:34Z" name: splitter-graph-b4182-58cdcd74d5-w67h4.18be43048fc3a9b0 namespace: kserve-ci-e2e-test resourceVersion: "30369" uid: 5efb4646-5d4d-414e-9cfc-e2cd14501022 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b4182-58cdcd74d5 namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: e73c2467-1754-42f0-bc16-ab6f00f66dde kind: Event lastTimestamp: "2026-07-01T19:54:51Z" message: 'Created pod: splitter-graph-b4182-58cdcd74d5-w67h4' metadata: creationTimestamp: "2026-07-01T19:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:51Z" name: splitter-graph-b4182-58cdcd74d5.18be4290a9a96de5 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: db8aecdf-5672-4d8c-8d74-26f1d1a01d21 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b4182 namespace: kserve-ci-e2e-test resourceVersion: "26659" uid: 2595d082-784e-4d0f-bd6e-21cc6cf3434b kind: Event lastTimestamp: "2026-07-01T19:54:51Z" message: Scaled up replica set splitter-graph-b4182-58cdcd74d5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:51Z" name: splitter-graph-b4182.18be4290a90c1573 namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 5b791dc8-02c7-48a4-85e7-93b55d7cdfab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b4182 namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: 7a25ff60-0566-45c6-acf8-0d193b4f5db7 kind: Event lastTimestamp: "2026-07-01T19:54:51Z" message: InferenceGraph [splitter-graph-b4182] is Ready metadata: creationTimestamp: "2026-07-01T19:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:51Z" name: splitter-graph-b4182.18be4290af067250 namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: 7b875502-76ed-45e1-aa1b-898468186668 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b4182 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: f98ec38e-2eea-43cf-97a0-5aed56013f91 kind: Event lastTimestamp: "2026-07-01T19:55:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:21Z" name: splitter-graph-b4182.18be42942797bb6d namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: e2bcd141-40ef-4498-bca7-4b8e5f6ec602 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b4182 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: f98ec38e-2eea-43cf-97a0-5aed56013f91 kind: Event lastTimestamp: "2026-07-01T19:55:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:21Z" name: splitter-graph-b4182.18be42942822d4ae namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: bf7a00f9-bcf9-4312-bc5b-5f3cd180378d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:34:01.101521Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-01ba2-predictor-547f47f477-gx5hl to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416d83aa86f1 namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: 46f87b74-117f-424d-881c-3cbe964205c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-01ba2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416d90ab357e namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 7754292a-1660-4167-abce-0e87cda0db04 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc12d65f3 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 7b2fc414-3126-4ebf-ae67-0f30651eb213 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc2e2f1df namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 81f4aff7-d791-4852-97bc-dfe0e01bb812 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc5fd6346 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 375db480-41ab-44f0-9e7c-1bd4382316fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc668144a namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: a875bae5-2726-4e85-831d-f59fd76edf9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc673d6a4 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: 00d8654e-3656-4455-90db-4f985d0df7ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc94173dd namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: fc7ec694-12f0-4621-a063-4ce686be0477 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:34:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:02Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dc9adf799 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 4dead21e-0fdf-45f6-b0a1-2f422c3c5542 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:43:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:19Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be416dfe7d14c0 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 0d0d4f91-3f41-4c7c-abf7-36492957a018 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:15Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be41eeb537e155 namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: 9c74640f-e41a-4df1-a64f-ad5c778436bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:15Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be41eeb5387bb0 namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: d199601f-4560-438e-a12e-e575dd94a32d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 4fd4a47d-0c8b-4867-b55f-3a1857a88604 kind: Event lastTimestamp: "2026-07-01T19:43:19Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:19Z" name: success-200-isvc-01ba2-predictor-547f47f477-gx5hl.18be41ef72d008b5 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 5ebb44b9-db8f-442e-8152-7afa8eea866f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-01ba2-predictor-547f47f477 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 4123bbfe-db69-4951-8a00-5a1ba41bfa6b kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'Created pod: success-200-isvc-01ba2-predictor-547f47f477-gx5hl' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-01ba2-predictor-547f47f477.18be416d83162b41 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 5e651d4f-b8bf-446c-9e3b-feb87154732f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: abf2e8b6-2920-46b7-a727-3496b6689e6e kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Scaled up replica set success-200-isvc-01ba2-predictor-547f47f477 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-01ba2-predictor.18be416d80bdb5e5 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 5390a4a5-59d7-4961-ac89-53490bbcddfe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 496c2b64-2325-4fc9-ba8d-58941581963d kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: success-200-isvc-01ba2-predictor.18be4170ff59df31 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 5e694980-5fbd-467c-ab0c-c92f259ff76a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 496c2b64-2325-4fc9-ba8d-58941581963d kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: success-200-isvc-01ba2-predictor.18be41710059e96d namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 87ebae38-e0a2-46ec-9ff7-d0c98ab8a629 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: 496c2b64-2325-4fc9-ba8d-58941581963d kind: Event lastTimestamp: "2026-07-01T19:35:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:16Z" name: success-200-isvc-01ba2-predictor.18be4177fef17acc namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: ecacd588-66eb-47f6-890d-6c081eff8137 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:34:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01ba2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: 496c2b64-2325-4fc9-ba8d-58941581963d kind: Event lastTimestamp: "2026-07-01T19:35:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:34:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:35:16Z" name: success-200-isvc-01ba2-predictor.18be4177ff65effb namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: 02b797b8-d807-42da-ab57-8e614506b2e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: e3c740f8-292d-4155-a5f2-aa37559a71dc kind: Event lastTimestamp: "2026-07-01T19:34:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:49Z" name: success-200-isvc-01ba2.18be416d80a01c31 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 56a2a7d1-2d3e-40a9-9113-41dfbc253066 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-01ba2 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: e3c740f8-292d-4155-a5f2-aa37559a71dc kind: Event lastTimestamp: "2026-07-01T19:34:49Z" message: InferenceService [success-200-isvc-01ba2] is Ready metadata: creationTimestamp: "2026-07-01T19:34:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:49Z" name: success-200-isvc-01ba2.18be4178baf4a791 namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: bd83fba2-357a-4885-aec3-72e6d1e72979 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:15.999448Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-46ddd-predictor-6647b68fc-zrszv to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:16Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eeb628fdc5 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 6ed94dc6-a72e-4d9c-a798-f378be168644 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-46ddd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:43:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:16Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eebf211473 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: e5b63917-27f8-4c3b-964d-f8b3441175a0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eef31d6609 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: ccff4aaf-e6da-48b8-8099-ed9f5776e372 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eef506e07e namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: b964213d-b9a3-4e22-ac17-2aed8a056819 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eef7dc8c90 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 504f00ba-63ac-4755-9345-cfdecf493e0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eef854f841 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: b6575fce-5d1d-4b1f-bb08-b41be1b8020b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eef85dcd20 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 801365a6-125c-4b56-b33e-9ccb8a00299d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eefbfb0746 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: b20c6bd5-2b8c-4b30-8613-e045105b149e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:17Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41eefc7fb739 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 3edd85db-1fae-4633-add8-8ebbadffa37b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:43:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:54Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41ef33947ea3 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 52037a11-4f97-489b-9cdb-3515a339ae43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41ff10be1394 namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 9937ee46-3089-4b67-b2a7-b6546efe6c88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41ff10bf0d18 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: aeed2a3b-77a9-4d85-8db0-8b71ce7ace88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 110eb4e9-8a92-4762-9350-7be4e8d78602 kind: Event lastTimestamp: "2026-07-01T19:44:29Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:29Z" name: success-200-isvc-46ddd-predictor-6647b68fc-zrszv.18be41ffbba511a6 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 4e6e4236-8aaf-4cb7-acbb-e2450ee09772 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-46ddd-predictor-6647b68fc namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: d317d52f-ec7a-41db-a16b-34d3a6287933 kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: 'Created pod: success-200-isvc-46ddd-predictor-6647b68fc-zrszv' metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:15Z" name: success-200-isvc-46ddd-predictor-6647b68fc.18be41eeb5b68d3f namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 2ab90be0-58e0-4c7a-bef8-3de9fb7ab8c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 0cd8ac6f-5b33-4ae9-bc17-5fb0cf64a619 kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: Scaled up replica set success-200-isvc-46ddd-predictor-6647b68fc from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:15Z" name: success-200-isvc-46ddd-predictor.18be41eeb3aa0299 namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: 1f52d420-44a6-4573-8f44-6bd54691c255 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 0bab4a87-d3fb-49c4-8985-33963bf2521a kind: Event lastTimestamp: "2026-07-01T19:43:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:30Z" name: success-200-isvc-46ddd-predictor.18be41f2322f7ad8 namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: ba37abbc-8cbb-4955-bed5-317c33693d67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 0bab4a87-d3fb-49c4-8985-33963bf2521a kind: Event lastTimestamp: "2026-07-01T19:43:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:30Z" name: success-200-isvc-46ddd-predictor.18be41f2328fa9fc namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 6a4e4c44-5abf-4b3b-9c73-d2e7a0124d57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:43:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: 0bab4a87-d3fb-49c4-8985-33963bf2521a kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:16Z" name: success-200-isvc-46ddd-predictor.18be41f5b13cf3eb namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 97d0774e-9c5f-45ba-9f0a-7fe06d8fac67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:43:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-46ddd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: 0bab4a87-d3fb-49c4-8985-33963bf2521a kind: Event lastTimestamp: "2026-07-01T19:44:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:16Z" name: success-200-isvc-46ddd-predictor.18be41f5b1b5cee7 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: e8bf6160-1aa6-4528-a613-5633b3144da1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 7412c40b-77fe-4a5f-bbf4-b62f1c633dea kind: Event lastTimestamp: "2026-07-01T19:44:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:04Z" name: success-200-isvc-46ddd.18be41eeb3c45c60 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: ca2efb65-7f80-4084-abd1-c8583218e765 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 7412c40b-77fe-4a5f-bbf4-b62f1c633dea kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-46ddd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-46ddd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:15Z" name: success-200-isvc-46ddd.18be41eeb47780ec namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 3fc4e7f4-9d01-49d7-89d9-9658b1b66cf2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 7412c40b-77fe-4a5f-bbf4-b62f1c633dea kind: Event lastTimestamp: "2026-07-01T19:43:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-46ddd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:15Z" name: success-200-isvc-46ddd.18be41eeb4783e68 namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 04ded1fe-1e4d-4d94-a2a3-f3a4cdff6807 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-46ddd namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 7412c40b-77fe-4a5f-bbf4-b62f1c633dea kind: Event lastTimestamp: "2026-07-01T19:44:04Z" message: InferenceService [success-200-isvc-46ddd] is Ready metadata: creationTimestamp: "2026-07-01T19:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:04Z" name: success-200-isvc-46ddd.18be41f9efc30f37 namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: c80e9f0b-5b94-4ace-918a-5b6b2f0af009 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:52:41.289813Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24825" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-74731-predictor-ddb44967c-gr2gj to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be427254175feb namespace: kserve-ci-e2e-test resourceVersion: "24843" uid: 3995602f-cd9a-4532-81ec-bc17766deeba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24853" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be42726dec3e0f namespace: kserve-ci-e2e-test resourceVersion: "24900" uid: 9ce7e949-6283-47b1-abc5-4aa405a4c8e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be42726f608e59 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: a8c942e9-bcab-44ef-a695-809a1a04e553 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be4272726e193e namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: 8cb2f528-5a33-495e-9066-102f52f38959 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be427272ea03e1 namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: ac1368e9-8adc-4743-9c5c-319ddf03c304 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be427272f4be0b namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 0977fb09-0e72-4cee-97e8-5c96e762ad80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be42727597176d namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: ed3c2000-12d5-4af3-9ded-f644bd4317b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be4272761778c4 namespace: kserve-ci-e2e-test resourceVersion: "24910" uid: 478a8337-988c-4da0-9d2d-8acc386ee400 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:52:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:53:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:19Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be4272bea6abe1 namespace: kserve-ci-e2e-test resourceVersion: "25214" uid: 2ae912c7-6afd-4f27-aa51-ec895517a74a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:51Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be4282bf04babd namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: 9b40ebd6-6965-4362-84ea-357c918d013a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:51Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be4282bf052951 namespace: kserve-ci-e2e-test resourceVersion: "26028" uid: e70aab9c-7712-4f2b-a5f4-2f541e4ec9ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-74731-predictor-ddb44967c-gr2gj namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: c11a248b-5d39-4a65-9a5c-9b7bff577cd8 kind: Event lastTimestamp: "2026-07-01T19:53:54Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:54Z" name: success-200-isvc-74731-predictor-ddb44967c-gr2gj.18be428346c3c08d namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: 70717e1c-c8d4-4bb1-bb6f-9c130490bb13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-74731-predictor-ddb44967c namespace: kserve-ci-e2e-test resourceVersion: "24812" uid: 9844663c-1c96-464d-9ea5-564e39805b0b kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Created pod: success-200-isvc-74731-predictor-ddb44967c-gr2gj' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor-ddb44967c.18be42725330b42b namespace: kserve-ci-e2e-test resourceVersion: "24830" uid: c701075f-affa-48ff-921f-b368a8f4a4c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "24810" uid: 84275abb-32aa-4b43-855d-1e411bd964f0 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Scaled up replica set success-200-isvc-74731-predictor-ddb44967c from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731-predictor.18be42724eb926f0 namespace: kserve-ci-e2e-test resourceVersion: "24813" uid: dc48e6e6-b292-49dd-9cc1-6a7c825efa58 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "24811" uid: db8924c8-7d38-4e7e-a87d-03f2157644da kind: Event lastTimestamp: "2026-07-01T19:53:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:26Z" name: success-200-isvc-74731-predictor.18be4275cdcafea6 namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: 2e10badb-9612-49d9-b47a-1a5e9bd0a9f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:52:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "24811" uid: db8924c8-7d38-4e7e-a87d-03f2157644da kind: Event lastTimestamp: "2026-07-01T19:53:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:26Z" name: success-200-isvc-74731-predictor.18be4275ce351054 namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: daef8946-701c-4343-b8e6-bdfefda72783 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: db8924c8-7d38-4e7e-a87d-03f2157644da kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-74731-predictor.18be42804afe2b86 namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: 49e0a322-ca30-4c12-b5db-5ec4f486cd8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-74731-predictor namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: db8924c8-7d38-4e7e-a87d-03f2157644da kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-74731-predictor.18be42804bb182b6 namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: d09ab2bd-a266-423f-b955-bd49da0d6bcf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: cc9f1d9e-5956-4ea8-9541-725b137ca675 kind: Event lastTimestamp: "2026-07-01T19:53:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:29Z" name: success-200-isvc-74731.18be42724ee373b7 namespace: kserve-ci-e2e-test resourceVersion: "25296" uid: a07c48e6-e107-4a19-ac1e-0f14b9b235a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: cc9f1d9e-5956-4ea8-9541-725b137ca675 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-74731": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-74731": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731.18be42724f64da3a namespace: kserve-ci-e2e-test resourceVersion: "24816" uid: 949dc49f-c54f-42b0-85f9-16e33be10be9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: cc9f1d9e-5956-4ea8-9541-725b137ca675 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-74731": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-74731.18be42724f6532cc namespace: kserve-ci-e2e-test resourceVersion: "24820" uid: 282d2e6f-95cc-4abe-a40e-775b3113a840 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-74731 namespace: kserve-ci-e2e-test resourceVersion: "25294" uid: cc9f1d9e-5956-4ea8-9541-725b137ca675 kind: Event lastTimestamp: "2026-07-01T19:53:29Z" message: InferenceService [success-200-isvc-74731] is Ready metadata: creationTimestamp: "2026-07-01T19:53:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:53:29Z" name: success-200-isvc-74731.18be427d7a2c9135 namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: b9f568b3-bb16-4bff-a77f-e6e9f6be7272 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:33:01.306753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415f979ec527 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: b92339e4-3502-4572-affd-fdab6646285d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-834d0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fa228447f namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b10f3cb3-e8f9-414c-a4a6-ea601ef0bbd2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fd4cce7f8 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 90756d8a-a8a9-47fa-9ddc-9123a1554967 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fd64484e8 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 1eab6723-218e-409b-a297-2c2db7a05c1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fd978297d namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: d7d0a965-93d6-4f88-ab7c-87e5ed9318ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fd9dd10a5 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 2a8d144c-fdbc-4dda-9753-8da8afbcd377 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fd9e728f7 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 51b32e3a-2a90-4952-870a-55811dda6f78 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fdcc32f6c namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 8332360c-b8ac-4f38-a3df-d2a2a8ec26ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:33:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:02Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415fdd2e3415 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: d90f10a6-5efd-4a21-972b-375e8850d35d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:42:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:18Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be415ff94c0a7e namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: d5baca44-09f9-4e9d-8eb1-4c01fe237db3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be41e0cdb9634d namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 03ec7cae-bae1-403b-b20e-9646506d9b20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be41e0cdbabfea namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 750b7ce6-6b6a-42e5-90a8-19438a08b5e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 52822648-3dee-484a-bc6f-6d982fcfa87f kind: Event lastTimestamp: "2026-07-01T19:42:18Z" message: 'Readiness probe failed: Get "https://10.133.0.17:8643/healthz": dial tcp 10.133.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:18Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l.18be41e16daf519a namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 63be3fca-a893-4d7f-a4c1-8a0784fbbfad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-834d0-predictor-6bdb44c5f9 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 1edf8614-c2b8-4781-931e-8195dc0b3464 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'Created pod: success-200-isvc-834d0-predictor-6bdb44c5f9-tdn8l' metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-834d0-predictor-6bdb44c5f9.18be415f96aab030 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 2cfaa8e4-584a-4302-9d1c-4364c5b0a497 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 2730fedf-ef85-48c9-a412-099c0ff35225 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Scaled up replica set success-200-isvc-834d0-predictor-6bdb44c5f9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-834d0-predictor.18be415f945ae268 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 014d42c7-3def-490c-9751-2dda51cc13c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 39840bb4-0ea0-449e-bc40-142a5e3affae kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-834d0-predictor.18be41631285b6b2 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: cb4b9c59-d29a-44fa-8e8d-3c6c06f4a52b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:33:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 39840bb4-0ea0-449e-bc40-142a5e3affae kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:33:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: success-200-isvc-834d0-predictor.18be41631300f788 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: f8829c36-8869-4122-928f-529beb600761 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 39840bb4-0ea0-449e-bc40-142a5e3affae kind: Event lastTimestamp: "2026-07-01T19:33:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:31Z" name: success-200-isvc-834d0-predictor.18be416692595b24 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 8237b307-7f54-4be3-a4f8-60ad515fef21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-834d0-predictor namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 39840bb4-0ea0-449e-bc40-142a5e3affae kind: Event lastTimestamp: "2026-07-01T19:33:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:31Z" name: success-200-isvc-834d0-predictor.18be416692dd1b4c namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: cddf1d7b-f57b-4546-8a2e-792fd4503015 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 4c1f186f-f461-48b2-bd46-4d0657d1a141 kind: Event lastTimestamp: "2026-07-01T19:33:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:49Z" name: success-200-isvc-834d0.18be415f94206ac6 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: ab352dae-1f8f-4354-803a-d29226b41a5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 4c1f186f-f461-48b2-bd46-4d0657d1a141 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-834d0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-834d0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-834d0.18be415f948a78f5 namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: a0743805-fbb3-4ff7-b1c7-2eabc7fd8fe3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 4c1f186f-f461-48b2-bd46-4d0657d1a141 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-834d0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-834d0.18be415f948acc33 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 80772e36-bde5-4493-8cc3-6045615b7d0e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 4c1f186f-f461-48b2-bd46-4d0657d1a141 kind: Event lastTimestamp: "2026-07-01T19:33:49Z" message: InferenceService [success-200-isvc-834d0] is Ready metadata: creationTimestamp: "2026-07-01T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:33:49Z" name: success-200-isvc-834d0.18be416ab4c3770b namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 82cebe63-49c6-42a4-95e6-f67e6e66c288 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:44:26.238778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-88843-predictor-694fdcfc59-krdsf to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff10c01e33 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 33929a4e-c31d-4741-a4c0-a38ebd852d68 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff2af88b53 namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: d19aebdc-6bf5-4ed9-8bc1-44baa12f5258 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff2c6324bd namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 86fe9c15-858b-4487-86b3-ef85f0326a6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff2f45cd4f namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 02ffa5a3-a350-43fd-a9b9-ea92c7fc8e67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff2fbe9e00 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: e6700be1-9df4-4369-bdeb-0274f3e3804c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff2fc9cd94 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 772e866f-fdc3-48e3-a86e-01e7b333c9d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff32cda6bc namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 2eb4f154-748c-49fe-b59c-896df3841e0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff33450b0f namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: 0dc1ea9b-0097-4212-9ede-d11245842492 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:53:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:44Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be41ff8f9d68e1 namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: c02dded9-a83b-48bc-92d7-4c137ccbfc49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be4280486f5dcd namespace: kserve-ci-e2e-test resourceVersion: "25459" uid: f3d461a5-1b14-4bec-9ef2-ced8ff4b0e8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be42804870786e namespace: kserve-ci-e2e-test resourceVersion: "25463" uid: 34fbca9b-c9d2-4051-ba9d-129825c58f82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88843-predictor-694fdcfc59-krdsf namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 13b71d55-6000-452c-8ebd-d8a92ac9b6e7 kind: Event lastTimestamp: "2026-07-01T19:53:44Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:44Z" name: success-200-isvc-88843-predictor-694fdcfc59-krdsf.18be4281042499a0 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 0377df34-a327-47bf-ab4d-606a65999648 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-88843-predictor-694fdcfc59 namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 77fdf9b1-0639-450f-b767-a7b7a52a2fa8 kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: 'Created pod: success-200-isvc-88843-predictor-694fdcfc59-krdsf' metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor-694fdcfc59.18be41ff1048b8f8 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: f8270dd9-adf4-4c40-a46c-2fd22126712c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: 3fd6912f-3698-4c93-9232-6f67592debfe kind: Event lastTimestamp: "2026-07-01T19:44:26Z" message: Scaled up replica set success-200-isvc-88843-predictor-694fdcfc59 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:26Z" name: success-200-isvc-88843-predictor.18be41ff0e7b068a namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: f2066f1f-2e4c-4af1-bd31-57432b815e19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 26420f5d-4cf6-455f-a507-c7649eac8be4 kind: Event lastTimestamp: "2026-07-01T19:45:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:11Z" name: success-200-isvc-88843-predictor.18be42028d502c8d namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 89a1600e-1b41-4ffd-abcb-78bab39a4e25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 26420f5d-4cf6-455f-a507-c7649eac8be4 kind: Event lastTimestamp: "2026-07-01T19:45:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:11Z" name: success-200-isvc-88843-predictor.18be42028dba2dac namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 9270a12c-13c0-4874-99d5-5f4c9ca909a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:44:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 26420f5d-4cf6-455f-a507-c7649eac8be4 kind: Event lastTimestamp: "2026-07-01T19:45:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:41Z" name: success-200-isvc-88843-predictor.18be42060ceb0529 namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: f6867846-a9a8-4571-9b97-5f7f93ab8ba0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:44:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88843-predictor namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 26420f5d-4cf6-455f-a507-c7649eac8be4 kind: Event lastTimestamp: "2026-07-01T19:45:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:45:41Z" name: success-200-isvc-88843-predictor.18be42060d4fecf0 namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: 1513e2a4-2552-4ebd-950e-25dfc052620c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-88843 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 5ca673d4-d63a-4c75-9caf-6861d7cbbc4d kind: Event lastTimestamp: "2026-07-01T19:45:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:45:14Z" name: success-200-isvc-88843.18be41ff0e7a07bd namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: b6bce936-a44a-47d3-849a-2dbaa13c7552 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:45:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-88843 namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 5ca673d4-d63a-4c75-9caf-6861d7cbbc4d kind: Event lastTimestamp: "2026-07-01T19:45:14Z" message: InferenceService [success-200-isvc-88843] is Ready metadata: creationTimestamp: "2026-07-01T19:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:45:14Z" name: success-200-isvc-88843.18be420a4b2aae31 namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: a0270d7c-7d69-4c8c-a000-61d311e43912 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:31:30.940970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:31:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:31:30Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414a8d6653d5 namespace: kserve-ci-e2e-test resourceVersion: "14130" uid: 8849e7c5-e3c4-4b2d-9100-34564321c3e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8fd9c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:31Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414a9747a335 namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 06196f0f-3def-4ef6-b9e5-9aa29269ed15 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:31Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:31:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:31:31Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414acaf484aa namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: 257372c0-2bdb-49df-8e29-22b42e2c9b7e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:32Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T19:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:32Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414ace0786ec namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 6b803c01-350f-4b9b-82b1-e511755599ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" in 12.298s (12.298s including waiting). Image size: 1335613554 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414dab196386 namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: 4f3a4205-b21e-4376-8818-194cb14d5e57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414dae464e86 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: d3a91989-40c3-4d08-933e-a873ace99c4d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414daeb821b2 namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: 54e04df9-9845-4d1b-aaa6-ffd899d75e61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T19:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:44Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414daec5830c namespace: kserve-ci-e2e-test resourceVersion: "14403" uid: ee63123c-fe70-434a-a506-66bb8057e680 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.237s (2.237s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:46Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414e34241eb3 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 8da65442-309d-4d02-ba32-bae8a7257d5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:46Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414e37803593 namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 29d0559a-a1c1-4a6e-827d-9ac520fb7683 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:31:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:31:46Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414e37ffe820 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 30159c73-97f1-4881-bfa8-c1aecc11293f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:31:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:32:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:24Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be414eaeac438d namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: e79507bb-bb17-46c9-8a9f-666cbf5493ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be415f95653e06 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: ce60fc72-84f8-41ac-b869-c43ab2e768e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 90646572-b409-4a64-b093-551a90805d07 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv.18be415f9565e324 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 4b3225f2-138e-4e72-898c-beb372aaddc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8fd9c-predictor-6c6b4fc89c namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: f272425c-bdcb-4fa6-bc1d-7b73d3bd0ab9 kind: Event lastTimestamp: "2026-07-01T19:31:30Z" message: 'Created pod: success-200-isvc-8fd9c-predictor-6c6b4fc89c-sbkzv' metadata: creationTimestamp: "2026-07-01T19:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:30Z" name: success-200-isvc-8fd9c-predictor-6c6b4fc89c.18be414a8c901e50 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 342a438f-c7c4-45e8-85b1-e8de53c8d4bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:31:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14111" uid: 6bbe15de-097a-4a6e-929b-281339d5ccb7 kind: Event lastTimestamp: "2026-07-01T19:31:30Z" message: Scaled up replica set success-200-isvc-8fd9c-predictor-6c6b4fc89c from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:31:30Z" name: success-200-isvc-8fd9c-predictor.18be414a802fedad namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 0568f42c-54df-4270-b0cf-a0c7670ec6b5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:31:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 418cd4e0-92f4-42ef-bcfd-630fe227dde2 kind: Event lastTimestamp: "2026-07-01T19:32:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:15Z" name: success-200-isvc-8fd9c-predictor.18be414e05f708c1 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f3117479-1bb5-43ed-917c-09a96698405c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:31:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 418cd4e0-92f4-42ef-bcfd-630fe227dde2 kind: Event lastTimestamp: "2026-07-01T19:32:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:15Z" name: success-200-isvc-8fd9c-predictor.18be414e0685f656 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 9b4d2847-ad11-4918-a486-7302371890b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:32:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 418cd4e0-92f4-42ef-bcfd-630fe227dde2 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-8fd9c-predictor.18be4151872f59b8 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 4793c291-dfbe-4840-993f-395aa3c4232f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:32:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8fd9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 418cd4e0-92f4-42ef-bcfd-630fe227dde2 kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:33:01Z" name: success-200-isvc-8fd9c-predictor.18be415188012378 namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: d1ac0760-90f5-4382-8271-6f5b377d30f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:31:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8fd9c namespace: kserve-ci-e2e-test resourceVersion: "14102" uid: 58ec2305-32bb-4005-80bc-45d04c68366c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-8fd9c.18be414a85eca225 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: b755825c-c499-4d77-8a2f-f0a723b67dee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8fd9c namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 58ec2305-32bb-4005-80bc-45d04c68366c kind: Event lastTimestamp: "2026-07-01T19:32:34Z" message: InferenceService [success-200-isvc-8fd9c] is Ready metadata: creationTimestamp: "2026-07-01T19:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:34Z" name: success-200-isvc-8fd9c.18be41596b8aae90 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: c27163dd-4ebd-4af1-93cc-e201747df203 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:42:16.236010Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0cbfb415d namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: c038e9b4-9347-4717-af97-261a296d4d7f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0e70b4dba namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: 5bd4ab93-41d6-4eb7-8328-b9a2c479358c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0e8783379 namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 9f22d2c4-ad83-4d8c-bee9-25a9fbe3bffe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0eb62aabf namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 2b11141f-c37b-4088-8888-a4450029e23e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0ebd60441 namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: e50f6262-8805-4b4d-a940-1d2e6245e0d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0ebe050c5 namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: 0dbf6bd5-8712-4947-81db-bb45f9442068 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0ee8347d4 namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 9d54e8c3-d664-4155-a4fc-fa0e0d5093db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e0eefa3122 namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: edbf6429-8026-43a4-9b50-1b9ee3501f5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:42:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:53Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41e12ea4a80f namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: 512a1187-0c52-4d83-a33d-d77464e19f3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41f12589cab0 namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: d7dfb3a6-44df-4ca7-bdfa-2a0726c2a2c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41f1258a6fdb namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: e3dab6b2-e57d-4f6b-a06e-0473590da9bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: e9acb56c-4c2e-48ff-9baf-86c9f9428d78 kind: Event lastTimestamp: "2026-07-01T19:43:28Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:28Z" name: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz.18be41f1b6c8cf7f namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 7d59f9c3-2fb5-4f2e-8567-1ec388152821 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ac147-predictor-6cd58fc96f namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 9f426b97-64b3-4286-bce8-b2dacd0ec85e kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Created pod: success-200-isvc-ac147-predictor-6cd58fc96f-b2hmz' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor-6cd58fc96f.18be41e0cb6b2d16 namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: d0761a73-6701-453f-abed-1187df91ee5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19540" uid: 85b9c2c1-c57a-4b48-92e4-b4a6d2f22082 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Scaled up replica set success-200-isvc-ac147-predictor-6cd58fc96f from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147-predictor.18be41e0c8247b1b namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 9f73a542-75f1-4f7b-9764-da1072b424d7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: 5347ea6f-5412-48a9-bb3d-2ef6a346d6b1 kind: Event lastTimestamp: "2026-07-01T19:42:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:46Z" name: success-200-isvc-ac147-predictor.18be41e4471d2590 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: c601b6e2-69fa-4409-9b96-f3eab481c367 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:42:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: 5347ea6f-5412-48a9-bb3d-2ef6a346d6b1 kind: Event lastTimestamp: "2026-07-01T19:42:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:42:46Z" name: success-200-isvc-ac147-predictor.18be41e447806546 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 948fd19d-2ee2-45c9-97e5-c68cf7c47fae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 5347ea6f-5412-48a9-bb3d-2ef6a346d6b1 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: success-200-isvc-ac147-predictor.18be41eb459ad58f namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: faa8cdf0-bf02-4041-b238-afc6a835bb1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac147-predictor namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 5347ea6f-5412-48a9-bb3d-2ef6a346d6b1 kind: Event lastTimestamp: "2026-07-01T19:43:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:16Z" name: success-200-isvc-ac147-predictor.18be41eb45ff081d namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: e149c5a7-3412-432c-b737-f3743b1ac08f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: cb1f0b36-9dc6-4044-83ea-d58179fa18c0 kind: Event lastTimestamp: "2026-07-01T19:43:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:04Z" name: success-200-isvc-ac147.18be41e0c842f98f namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 6b3662ca-87bb-47ca-9fbd-c342b76db58d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: cb1f0b36-9dc6-4044-83ea-d58179fa18c0 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-ac147": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ac147": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147.18be41e0c8a82103 namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: 8593bee8-7b47-4bf1-98c6-672f5a100876 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: cb1f0b36-9dc6-4044-83ea-d58179fa18c0 kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ac147": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:42:16Z" name: success-200-isvc-ac147.18be41e0c8a8818a namespace: kserve-ci-e2e-test resourceVersion: "19548" uid: 56987f4f-828f-4bba-b577-75a0743acf68 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac147 namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: cb1f0b36-9dc6-4044-83ea-d58179fa18c0 kind: Event lastTimestamp: "2026-07-01T19:43:03Z" message: InferenceService [success-200-isvc-ac147] is Ready metadata: creationTimestamp: "2026-07-01T19:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:03Z" name: success-200-isvc-ac147.18be41ebe9b9a67a namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 863f668d-a6d4-49ec-b491-f2dead43cd07 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:51.617002Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b4182-predictor-58995c77cf-8brpg to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:51Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282b3eb1d48 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: ee0f568b-5733-49fe-95ce-0416db48b5f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25986" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282cde83c2c namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: 2e5a509e-7252-4115-9586-b4add4da1ef8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282cf4606ea namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 2cbe8db9-40a1-46fb-92c0-74fcc27ddd92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282d25f62e1 namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: a83c55e7-b464-4294-9840-59e0cecb6820 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282d2dcfbbe namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 2546d7bf-4edf-40c4-ac58-e5572e0d8aa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282d2e73ee0 namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: a7ea79d8-908c-4ecd-bfe3-05d738cd7010 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282d59ca110 namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: 0f049dd0-f2b8-4644-82d2-4e02744ee1ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T19:53:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:52Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282d612f3ed namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: bcfa271e-e982-4077-9f4a-51a8ecb40eb3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:53:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T20:03:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:08Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4282dee14df0 namespace: kserve-ci-e2e-test resourceVersion: "30175" uid: 43d7e715-8aa4-4d9d-bc59-fa7364c92c47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T20:03:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:06Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4303e1683953 namespace: kserve-ci-e2e-test resourceVersion: "30145" uid: 5701ed64-fff0-40d6-9f0e-91d31af89470 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T20:03:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:06Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4303e1689d10 namespace: kserve-ci-e2e-test resourceVersion: "30144" uid: 2339733f-db30-42b3-98ab-344e1ea53f36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b4182-predictor-58995c77cf-8brpg namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 0e0a9972-6658-46fe-b0ce-20604af45554 kind: Event lastTimestamp: "2026-07-01T20:03:08Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:03:08Z" name: success-200-isvc-b4182-predictor-58995c77cf-8brpg.18be4304533835f0 namespace: kserve-ci-e2e-test resourceVersion: "30174" uid: 5a2df687-826c-461b-a996-277d0736d1cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b4182-predictor-58995c77cf namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 6b2b6718-6abb-4ec8-abe5-5ce74bf78e34 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: 'Created pod: success-200-isvc-b4182-predictor-58995c77cf-8brpg' metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:51Z" name: success-200-isvc-b4182-predictor-58995c77cf.18be4282b32ec2b9 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: f169b58b-634e-4e74-aca2-cf55197525e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: ba8dd49a-0211-41df-b632-5d7e0c306ac1 kind: Event lastTimestamp: "2026-07-01T19:53:51Z" message: Scaled up replica set success-200-isvc-b4182-predictor-58995c77cf from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:51Z" name: success-200-isvc-b4182-predictor.18be4282b1576164 namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 05969a0e-6a85-4a17-96e6-0b01a444e5a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: b59e6122-aeee-4797-9b1d-b08581c7f716 kind: Event lastTimestamp: "2026-07-01T19:54:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:06Z" name: success-200-isvc-b4182-predictor.18be42862fd43951 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: 5eb7191e-b6c1-4d6c-a97f-ab11db1e3aaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: b59e6122-aeee-4797-9b1d-b08581c7f716 kind: Event lastTimestamp: "2026-07-01T19:54:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:06Z" name: success-200-isvc-b4182-predictor.18be42863035392f namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: b77ed276-07a5-4078-b7be-72d7e4753145 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:54:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: b59e6122-aeee-4797-9b1d-b08581c7f716 kind: Event lastTimestamp: "2026-07-01T19:54:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:51Z" name: success-200-isvc-b4182-predictor.18be4289af82fe73 namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: a15d69bc-54fb-4c19-afef-5757abb03857 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:54:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b4182-predictor namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: b59e6122-aeee-4797-9b1d-b08581c7f716 kind: Event lastTimestamp: "2026-07-01T19:54:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:51Z" name: success-200-isvc-b4182-predictor.18be4289afda0730 namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 24c3800d-ad4c-4321-9c22-0795a748c8ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:53:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b4182 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 9ddf1d1f-9165-4750-89c0-c934e2a7d5f2 kind: Event lastTimestamp: "2026-07-01T19:54:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:38Z" name: success-200-isvc-b4182.18be4282b158f077 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: b5c4c1ad-9e8f-40d2-9342-5b12aa9878fb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b4182 namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: 9ddf1d1f-9165-4750-89c0-c934e2a7d5f2 kind: Event lastTimestamp: "2026-07-01T19:54:38Z" message: InferenceService [success-200-isvc-b4182] is Ready metadata: creationTimestamp: "2026-07-01T19:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:38Z" name: success-200-isvc-b4182.18be428d9ab8b8cd namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 8a8c2392-28e4-419c-834b-0c1180fb864a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:43:26.482983Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f127070c97 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8fd453f4-c24c-46b1-ac97-87af9fca0831 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c6a40-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f135f96a7e namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 85563e06-1458-417a-acce-6d1696dfba1b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f164c7b3c2 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 6894ec9b-cc7d-4bcd-9fd4-333e2c7f0f9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f16640d95b namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 30fb5baa-b9cb-4531-bea5-d80f2aba7d1f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f169109db4 namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: bc0f7079-a48e-46db-89a0-5b552675f2ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f169a39b90 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 4cb0318f-09a2-454e-91cc-98f447f17f74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f169add2b6 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 150bbf0b-9a87-445f-ada4-40f35e43e73a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f16ca33323 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: dd50dccc-eb73-40d8-a4fc-f2e4b848d231 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:43:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:43:27Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f16d1e7e16 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: eeb3ffa5-c9c5-4d5d-b30f-2c1e45493cae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:52:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:44Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be41f18a07947b namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 07b41e84-4fd6-4a4e-b6f8-1f9786946705 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be427255beef84 namespace: kserve-ci-e2e-test resourceVersion: "24854" uid: 99ea3880-15c7-4c63-9f8c-9c83ea4cd475 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:52:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:41Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be427255c08220 namespace: kserve-ci-e2e-test resourceVersion: "24857" uid: e5ae94d6-45f6-4c8c-b3bc-077846579c19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:52:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 7f7ebb07-09db-4810-ac21-cb4ff6867654 kind: Event lastTimestamp: "2026-07-01T19:52:44Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:52:44Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh.18be4272fe728b0f namespace: kserve-ci-e2e-test resourceVersion: "24941" uid: de27a00d-b453-4179-951c-1141e0ef70ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c6a40-predictor-759bb4bdbf namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 42c5c617-5834-4961-9f9f-c9290e47e6ad kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'Created pod: success-200-isvc-c6a40-predictor-759bb4bdbf-7p8wh' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-c6a40-predictor-759bb4bdbf.18be41f12642d320 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: dfedd181-3e98-41cc-82cd-62a5a0d5457d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: e0bacfc2-612d-4344-821b-20748174d7b9 kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: Scaled up replica set success-200-isvc-c6a40-predictor-759bb4bdbf from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-c6a40-predictor.18be41f1240c606f namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: 7028b74f-4e5a-4952-a236-ef3e2dabd5c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: e0bf64f1-321c-4b49-90c1-0fdf460cda4f kind: Event lastTimestamp: "2026-07-01T19:44:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:11Z" name: success-200-isvc-c6a40-predictor.18be41f4a2bb1080 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: be75714b-c027-450f-a17f-ec6eb7d8af02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:43:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: e0bf64f1-321c-4b49-90c1-0fdf460cda4f kind: Event lastTimestamp: "2026-07-01T19:44:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:43:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:11Z" name: success-200-isvc-c6a40-predictor.18be41f4a3218410 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 13934207-ccc0-40f5-a078-e581627e69fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:43:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: e0bf64f1-321c-4b49-90c1-0fdf460cda4f kind: Event lastTimestamp: "2026-07-01T19:44:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:56Z" name: success-200-isvc-c6a40-predictor.18be41f822231fe8 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 518d3ec8-4ff4-4339-97be-0d45a398d7ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:43:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6a40-predictor namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: e0bf64f1-321c-4b49-90c1-0fdf460cda4f kind: Event lastTimestamp: "2026-07-01T19:44:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:44:56Z" name: success-200-isvc-c6a40-predictor.18be41f82286ecb3 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 3ce311be-6754-4b18-87c1-55b2444896ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 21f3953f-c9e7-4601-a90c-388279a0561f kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:14Z" name: success-200-isvc-c6a40.18be41f124354db2 namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 8e4453b5-6217-49a5-bde7-054642579aef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 21f3953f-c9e7-4601-a90c-388279a0561f kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-c6a40": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c6a40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-c6a40.18be41f1253a8a3c namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: eabb4819-e142-4904-9484-fac5a62b47f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:43:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 21f3953f-c9e7-4601-a90c-388279a0561f kind: Event lastTimestamp: "2026-07-01T19:43:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c6a40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:43:26Z" name: success-200-isvc-c6a40.18be41f1253adbb3 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 6a01f290-a0d0-4849-88b2-249d53129ed3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:44:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c6a40 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 21f3953f-c9e7-4601-a90c-388279a0561f kind: Event lastTimestamp: "2026-07-01T19:44:14Z" message: InferenceService [success-200-isvc-c6a40] is Ready metadata: creationTimestamp: "2026-07-01T19:44:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:44:14Z" name: success-200-isvc-c6a40.18be41fc455be805 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 93ac44fc-65ae-4322-bd48-d7150aaa95cc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:53:41.208993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25441" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 to ip-10-0-129-40.ec2.internal metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be4280478d9c45 namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: 9c918b82-6692-4369-bdf1-f0b47b11cefb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25464" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be428061dfbeea namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 75e7f3f3-f761-489a-bd7c-cfd54b0408e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be4280637a8eae namespace: kserve-ci-e2e-test resourceVersion: "25540" uid: e594dc3a-e01a-426c-95b0-621e75ff5f38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be4280667da342 namespace: kserve-ci-e2e-test resourceVersion: "25542" uid: bf3cf14e-1740-4ad8-a4ae-23ccb8f88fff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be428066f3e47b namespace: kserve-ci-e2e-test resourceVersion: "25543" uid: 1c6582ad-fe4d-4436-bd8e-65a736a3aa81 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be428066fe4359 namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: cb61ac38-e904-442a-8070-adcafd8e7c24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be428069797384 namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: 6d753f14-78d0-4b87-a118-e167f27e8f90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be428069f19791 namespace: kserve-ci-e2e-test resourceVersion: "25549" uid: 50bc31d9-dd26-4c12-a4a8-d05ff33c937d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T19:54:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:18Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be4280882a6592 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: f0e3e4f5-8f2c-4d48-bc0e-c209946fc0bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T20:11:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:00Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be43725339ca83 namespace: kserve-ci-e2e-test resourceVersion: "33431" uid: e105637e-137d-46dd-bdda-f08136b91775 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T20:11:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:00Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be4372533a4fd2 namespace: kserve-ci-e2e-test resourceVersion: "33432" uid: 91c74d2e-e5f3-4242-bf47-d582db185036 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07cd3f38-e097-4fe2-b446-f6875dbd0ab9 kind: Event lastTimestamp: "2026-07-01T20:11:03Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:03Z" name: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95.18be4372e8efdef2 namespace: kserve-ci-e2e-test resourceVersion: "33502" uid: ed6b4b67-f0a4-4d5e-81df-cf76b2ea89fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-40.ec2.internal source: component: kubelet host: ip-10-0-129-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ee34f-predictor-6bdb89f555 namespace: kserve-ci-e2e-test resourceVersion: "25425" uid: f7a8382f-9094-40ab-9e6c-994625faf7fc kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: 'Created pod: success-200-isvc-ee34f-predictor-6bdb89f555-k5d95' metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor-6bdb89f555.18be4280470f49ea namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: 74be41e6-8b9d-49d6-8fc4-d8474d4b09ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25422" uid: a3d8d56f-18c2-4814-ae01-05ff09acf872 kind: Event lastTimestamp: "2026-07-01T19:53:41Z" message: Scaled up replica set success-200-isvc-ee34f-predictor-6bdb89f555 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:41Z" name: success-200-isvc-ee34f-predictor.18be4280442b2e11 namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 1c453973-2e24-4492-a149-c79d17a96378 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: 363068cd-fe57-47f6-995b-64098e2060d1 kind: Event lastTimestamp: "2026-07-01T19:53:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:56Z" name: success-200-isvc-ee34f-predictor.18be4283c2e330c0 namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: 9477304e-32c9-4a7e-ad3d-a1ad074ef958 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: 363068cd-fe57-47f6-995b-64098e2060d1 kind: Event lastTimestamp: "2026-07-01T19:53:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:53:56Z" name: success-200-isvc-ee34f-predictor.18be4283c35778b8 namespace: kserve-ci-e2e-test resourceVersion: "26121" uid: 62ebea2f-fe27-4cd6-a264-2e7fba0a6558 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:54:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 363068cd-fe57-47f6-995b-64098e2060d1 kind: Event lastTimestamp: "2026-07-01T19:55:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:11Z" name: success-200-isvc-ee34f-predictor.18be428742aa5c27 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: d5f8526a-ddc8-4a04-89e2-c5c3ee5544e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:54:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ee34f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 363068cd-fe57-47f6-995b-64098e2060d1 kind: Event lastTimestamp: "2026-07-01T19:55:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:11Z" name: success-200-isvc-ee34f-predictor.18be42874378b755 namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: e88fec7a-cfe2-4e47-b5ba-e7a9dc62ec2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ee34f namespace: kserve-ci-e2e-test resourceVersion: "25405" uid: 95f8907b-c476-4f8f-8b66-315514c60630 kind: Event lastTimestamp: "2026-07-01T19:54:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:28Z" name: success-200-isvc-ee34f.18be428044324d6d namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 45d11385-a81a-4912-89a7-f29183e4d1fa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: 95f8907b-c476-4f8f-8b66-315514c60630 kind: Event lastTimestamp: "2026-07-01T19:54:28Z" message: InferenceService [success-200-isvc-ee34f] is Ready metadata: creationTimestamp: "2026-07-01T19:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:28Z" name: success-200-isvc-ee34f.18be428b46dc39cd namespace: kserve-ci-e2e-test resourceVersion: "26409" uid: c714d463-8533-4366-9ac7-77c0ff7bcfbd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:34:01.363421Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-834d0-6d7b4cc848-b2hn5 to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be416d9346da94 namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 8e52451c-bf61-4b0c-bbf7-448ccb08a002 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be416dad25539b namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: f8cb5bdf-fd5c-4db0-8ad3-678a12118424 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-834d0} kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be416daf4f1452 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: d51e406e-a2e6-48c9-a13a-ae33723394e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-834d0} kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'Created container: switch-graph-834d0' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be416db2a1f64e namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 93f1c267-2c06-4d10-a2d6-165a3609ec06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-834d0} kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Started container switch-graph-834d0 metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be416db319e24c namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 6a018472-fd5f-4a7e-ba37-59d1fa5401a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-834d0} kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: "2026-07-01T19:42:16Z" message: Stopping container switch-graph-834d0 metadata: creationTimestamp: "2026-07-01T19:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:16Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be41e0bfffc5c5 namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 9bf1fdf8-cab0-4a98-ae9f-8ad3ced4b41b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-834d0} kind: Pod name: switch-graph-834d0-6d7b4cc848-b2hn5 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 757e1922-1aea-4ea4-b1f5-193de2c2eabb kind: Event lastTimestamp: "2026-07-01T19:42:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:42:43Z" name: switch-graph-834d0-6d7b4cc848-b2hn5.18be41e165b2390c namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 192c397c-fb61-4433-aef6-89e66c9e3db4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-834d0-6d7b4cc848 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: ac3b79a5-b348-4b71-810b-83c8d8e14642 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: 'Created pod: switch-graph-834d0-6d7b4cc848-b2hn5' metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0-6d7b4cc848.18be416d9291c73c namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: af974da1-c444-462b-ae4b-019641e33174 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 5b5671af-a7f8-4097-80ea-df2f114642b3 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: Scaled up replica set switch-graph-834d0-6d7b4cc848 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0.18be416d91cf8153 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: e0a65f6e-dbb7-49e0-81bd-36641980eec6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:34:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 5ddb5ff2-7fc8-4384-928a-57177a35fc44 kind: Event lastTimestamp: "2026-07-01T19:34:01Z" message: InferenceGraph [switch-graph-834d0] is Ready metadata: creationTimestamp: "2026-07-01T19:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:34:01Z" name: switch-graph-834d0.18be416d9644fa1d namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: d64f9d62-277e-417a-9f30-5eb2ad701cb6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 95f08259-682b-44e8-93dc-625019a90c14 kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: switch-graph-834d0.18be417110aa429d namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 3b1a4bdc-a26d-4750-8e47-43f83392ee35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-834d0 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 95f08259-682b-44e8-93dc-625019a90c14 kind: Event lastTimestamp: "2026-07-01T19:34:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:34:31Z" name: switch-graph-834d0.18be4171111a8d33 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: 793e0443-1038-45c9-8353-a1600934491d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:32:50.760786Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8fd9c-6bb6b4fcdb-4b2kd to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:32:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:32:50Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415d230832bb namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: ca51a993-6014-42f5-8253-a1197b608796 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:32:51Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:32:51Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415d3cb5892b namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 8352ac8e-fd7b-4d3d-8108-2569d0622191 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fd9c} kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:32:51Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" metadata: creationTimestamp: "2026-07-01T19:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:51Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415d3e8d5816 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 14f59e3b-3bbb-4445-b8af-5dd9891bf2d6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fd9c} kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:32:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" in 2.534s (2.534s including waiting). Image size: 218070141 bytes.' metadata: creationTimestamp: "2026-07-01T19:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:53Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415dd5a556ed namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 8fa3cb2a-8820-41f1-995a-da7523f155eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fd9c} kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:32:53Z" message: 'Created container: switch-graph-8fd9c' metadata: creationTimestamp: "2026-07-01T19:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:53Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415dd89d1963 namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: f48d0427-c63c-4328-82a7-b13658606563 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fd9c} kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:32:53Z" message: Started container switch-graph-8fd9c metadata: creationTimestamp: "2026-07-01T19:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:32:53Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415dd90ed438 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: e13882b8-4e3a-445c-a793-74a2526815eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:33:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fd9c} kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:33:01Z" message: Stopping container switch-graph-8fd9c metadata: creationTimestamp: "2026-07-01T19:33:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:01Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be415f897e693e namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: d877ec5c-e337-4834-ae1c-e9e2059d2c34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8fd9c} kind: Pod name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 09ba7a20-6f4f-444d-8471-fd4cabf42b7a kind: Event lastTimestamp: "2026-07-01T19:33:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:33:30Z" name: switch-graph-8fd9c-6bb6b4fcdb-4b2kd.18be41609203a312 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: a510d70e-664b-4a49-a4a3-871088ba18d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8fd9c-6bb6b4fcdb namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 57fa6630-ce48-45f4-96b7-161c8302304c kind: Event lastTimestamp: "2026-07-01T19:32:50Z" message: 'Created pod: switch-graph-8fd9c-6bb6b4fcdb-4b2kd' metadata: creationTimestamp: "2026-07-01T19:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:50Z" name: switch-graph-8fd9c-6bb6b4fcdb.18be415d2276cb66 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 3e023642-a143-42b0-afba-60677c7c2641 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8fd9c namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 203e6c2f-4661-40da-b518-ad93104ee3c8 kind: Event lastTimestamp: "2026-07-01T19:32:50Z" message: Scaled up replica set switch-graph-8fd9c-6bb6b4fcdb from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:32:50Z" name: switch-graph-8fd9c.18be415d2129ebd2 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 3cdc0a11-7c3e-4f46-9a4e-8e31fb91a984 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:32:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8fd9c namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 2e6ab158-41d1-4934-80b9-ed6148219d12 kind: Event lastTimestamp: "2026-07-01T19:32:50Z" message: InferenceGraph [switch-graph-8fd9c] is Ready metadata: creationTimestamp: "2026-07-01T19:32:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:32:50Z" name: switch-graph-8fd9c.18be415d27f2f6c2 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: ad2a6d78-35c6-4374-bf67-3d029a51529a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:54:41.247609Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ee34f-5765b8d4c6-r9x5w to ip-10-0-138-252.ec2.internal metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be428e42221dee namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: adf6859e-d983-4ae6-ab40-3ab4aaad75f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-dc764c5f5-wwd54 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be428e5bc80d7b namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 1a115bda-2c0a-4ae0-88b6-9c7df31f0134 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ee34f} kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:739b6aa792fe51f6eef0c708781538ef463424156c64441d55ba532852ef9645" already present on machine metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be428e5d07afc1 namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: e3ce11fd-aa6e-4ee1-b509-97d083a2d4b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ee34f} kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: 'Created container: switch-graph-ee34f' metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be428e60852cf5 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 1ce79da5-2de8-48e6-8602-1f99d7555ce3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ee34f} kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: Started container switch-graph-ee34f metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be428e60fd5bf7 namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: d2ff711b-2975-40ea-a080-32740eb7332c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ee34f} kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: "2026-07-01T20:11:00Z" message: Stopping container switch-graph-ee34f metadata: creationTimestamp: "2026-07-01T20:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:00Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be437249bae516 namespace: kserve-ci-e2e-test resourceVersion: "33414" uid: 975ed99e-c989-45a0-8b63-1b21263615ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T20:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ee34f} kind: Pod name: switch-graph-ee34f-5765b8d4c6-r9x5w namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: fc929f5f-8d42-4360-9329-a4b81c206f7f kind: Event lastTimestamp: "2026-07-01T20:11:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:11:28Z" name: switch-graph-ee34f-5765b8d4c6-r9x5w.18be4372e96c0984 namespace: kserve-ci-e2e-test resourceVersion: "33686" uid: a2697dc1-059b-4e06-be65-8a83bcb69119 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-252.ec2.internal source: component: kubelet host: ip-10-0-138-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ee34f-5765b8d4c6 namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: 607adfa5-5631-46c6-a838-866615d0df34 kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: 'Created pod: switch-graph-ee34f-5765b8d4c6-r9x5w' metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f-5765b8d4c6.18be428e41777ef9 namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: dd6a1fbf-4354-4c2a-9cec-7b56070afbbc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: e5aa4aa3-ad00-4135-9041-b6ba398a1468 kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: Scaled up replica set switch-graph-ee34f-5765b8d4c6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f.18be428e407546fa namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: 9df6e628-b83e-4a50-9b6a-7414f6f1a73b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 16a2875e-4907-4701-896d-4c6e39b98df8 kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: InferenceGraph [switch-graph-ee34f] is Ready metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f.18be428e4a7801b5 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 511326fb-6fa6-477a-b5a2-90d31e8c756c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: 16a2875e-4907-4701-896d-4c6e39b98df8 kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: 'Failed to update status for InferenceGraph "switch-graph-ee34f": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-ee34f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f.18be428e4dbb09bf namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: fc116661-973f-44e9-b5c7-2c6430aa73ae reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: 16a2875e-4907-4701-896d-4c6e39b98df8 kind: Event lastTimestamp: "2026-07-01T19:54:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-ee34f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:54:41Z" name: switch-graph-ee34f.18be428e4dbb6744 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: e5013aa1-742c-4027-b77c-dc06af34836d reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 58751243-60a0-46d1-ac37-47d1c0a32170 kind: Event lastTimestamp: "2026-07-01T19:54:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:56Z" name: switch-graph-ee34f.18be4291beee2a27 namespace: kserve-ci-e2e-test resourceVersion: "26727" uid: c8436f27-92cb-4e22-b614-06980337b703 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:54:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 58751243-60a0-46d1-ac37-47d1c0a32170 kind: Event lastTimestamp: "2026-07-01T19:54:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:54:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:54:56Z" name: switch-graph-ee34f.18be4291bf4f0d56 namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: 3063d65a-0217-40c6-8df2-20609bb4d33f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 58751243-60a0-46d1-ac37-47d1c0a32170 kind: Event lastTimestamp: "2026-07-01T19:55:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:11Z" name: switch-graph-ee34f.18be42953e228ef1 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 6c41e561-0e4e-4d43-8e5b-5a4112e9581a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ee34f namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: 58751243-60a0-46d1-ac37-47d1c0a32170 kind: Event lastTimestamp: "2026-07-01T19:55:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:55:11Z" name: switch-graph-ee34f.18be42953e865d68 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: b2f5b376-026f-48aa-b326-3fa1b86f212e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33689"