--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:36.978624Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b2e49-5bbbdc7c77-q8pp5 to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:36Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a961818a9da29b namespace: kserve-ci-e2e-test resourceVersion: "19341" uid: 2bf4b815-2cce-4bb4-bec4-9b875bf2157d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:37Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:37Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a96181a36ca9d2 namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 2bfc6b8d-e46b-4866-9e99-d40ff137300b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e49} kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:37Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a96181a49dc0f1 namespace: kserve-ci-e2e-test resourceVersion: "19362" uid: c343183f-deff-434d-938e-531526923b79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e49} kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: 'Created container: ensemble-graph-b2e49' metadata: creationTimestamp: "2026-04-24T19:25:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:37Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a96181a784f45d namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: 7c0ac743-ef2d-4832-bab9-3de9333da713 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e49} kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: Started container ensemble-graph-b2e49 metadata: creationTimestamp: "2026-04-24T19:25:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:37Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a96181a7f5ea23 namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: 013d8383-a6ef-4c84-ae52-0628e5f81b57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e49} kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container ensemble-graph-b2e49 metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a96183e3d65d71 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 6de49591-6525-4a40-9388-c95e918b78e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e49} kind: Pod name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 7acf4d57-716d-4c7f-acaa-636ce2b36f0d kind: Event lastTimestamp: "2026-04-24T19:26:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:13Z" name: ensemble-graph-b2e49-5bbbdc7c77-q8pp5.18a96184495cff4f namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: 82c9fa49-4212-4e74-8a5f-c80b54d0293f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b2e49-5bbbdc7c77 namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: 8ae1a6b7-cf0e-4eda-ad05-ccdddb5f08ec kind: Event lastTimestamp: "2026-04-24T19:25:36Z" message: 'Created pod: ensemble-graph-b2e49-5bbbdc7c77-q8pp5' metadata: creationTimestamp: "2026-04-24T19:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:36Z" name: ensemble-graph-b2e49-5bbbdc7c77.18a961818a368a24 namespace: kserve-ci-e2e-test resourceVersion: "19338" uid: f24da7b1-1fa1-40b9-ab25-cb56bd4622a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 56580683-63d0-42ea-8af3-eb4b8a51a771 kind: Event lastTimestamp: "2026-04-24T19:25:36Z" message: Scaled up replica set ensemble-graph-b2e49-5bbbdc7c77 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:36Z" name: ensemble-graph-b2e49.18a9618189827385 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: 0f93340a-7914-4bcd-aa87-a2ac14d55293 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "19350" uid: 0e287ecc-e175-48d7-9cd1-39373b1c7b9f kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: InferenceGraph [ensemble-graph-b2e49] is Ready metadata: creationTimestamp: "2026-04-24T19:25:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:37Z" name: ensemble-graph-b2e49.18a961818eb184e1 namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: 67454388-c56a-42c0-8f07-2369757a7744 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:26:57.277226Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ec538-7b5b96995b-hjrb2 to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a961943cc90a85 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: c050f647-9c61-41ed-aeea-50f74d1807cf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a9619455964e7b namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 4c616e5b-7730-4a4b-b777-61b2600e25b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ec538} kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a961945750ae7b namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: ecfaf4c7-ebb0-45e2-b378-f493fcc1d5c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ec538} kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: 'Created container: ensemble-graph-ec538' metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a961945a69ee12 namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: b651eb74-05ba-49cb-a021-5e9f889bdb5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ec538} kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: Started container ensemble-graph-ec538 metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a961945ada0548 namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: b1cfab39-1f7a-41eb-8987-fcfc033799c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ec538} kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Stopping container ensemble-graph-ec538 metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a962076ed08dd1 namespace: kserve-ci-e2e-test resourceVersion: "23902" uid: 01acbae2-2743-4960-83e7-766008c0cf02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ec538} kind: Pod name: ensemble-graph-ec538-7b5b96995b-hjrb2 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: dcf65dd5-f28c-4130-aed1-b49645bf4a43 kind: Event lastTimestamp: "2026-04-24T19:35:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:39Z" name: ensemble-graph-ec538-7b5b96995b-hjrb2.18a96207e45c9822 namespace: kserve-ci-e2e-test resourceVersion: "24234" uid: 7d33df6e-009f-4428-89c2-bd1551df2085 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ec538-7b5b96995b namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: c3434e0c-04ea-41ba-b168-61911306bccb kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: 'Created pod: ensemble-graph-ec538-7b5b96995b-hjrb2' metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538-7b5b96995b.18a961943c56ae78 namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: bb25fd11-295c-4b21-b5f2-ac029b48110a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ec538 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 36495407-83ea-43c4-8209-1b6127a50f4a kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: Scaled up replica set ensemble-graph-ec538-7b5b96995b from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538.18a961943ba2575b namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: d6082924-a71f-4a55-9dd2-4efae2e41d6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ec538 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: 2969af24-8b9f-4f51-9e1e-66dc91db6235 kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: InferenceGraph [ensemble-graph-ec538] is Ready metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:57Z" name: ensemble-graph-ec538.18a96194409e7a8e namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: f04b7096-23dc-469a-8a88-37ab9fb8d5af reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-ec538 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 0082b1e0-4b63-4105-8764-d4d079b7d3a1 kind: Event lastTimestamp: "2026-04-24T19:27:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:57Z" name: ensemble-graph-ec538.18a96197ba71ab84 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 271d8d27-9c9a-4bd8-a4d5-157fb87d55b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-ec538 namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 0082b1e0-4b63-4105-8764-d4d079b7d3a1 kind: Event lastTimestamp: "2026-04-24T19:27:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:57Z" name: ensemble-graph-ec538.18a96197bab7048a namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 4bfef691-cf4c-44e4-a141-63528987098a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:36:22.538923Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24884" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-11f3c-predictor-5b494597f6-5r9st to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a96217d9020333 namespace: kserve-ci-e2e-test resourceVersion: "24888" uid: af37aca7-2eb5-44fe-b68b-2eb16d7eb4a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-11f3c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a96217e006fc99 namespace: kserve-ci-e2e-test resourceVersion: "24895" uid: 70dfe01b-5d6e-4335-85c9-ad2d3f2b112f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24893" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a9621816393796 namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 96458016-c938-4bbc-9b4d-83a389953d68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a96218179dc5ab namespace: kserve-ci-e2e-test resourceVersion: "24920" uid: 34b36d73-998a-4cf2-8e41-d5cce42f685d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962181a7dfd80 namespace: kserve-ci-e2e-test resourceVersion: "24923" uid: 1e7ca145-1373-422e-9a52-ebb6fbcfdb9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962181afabf74 namespace: kserve-ci-e2e-test resourceVersion: "24924" uid: e0b7a515-f3fa-42af-9860-3fe9d3670432 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962181b06fce3 namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: ab5964b0-bcbd-4488-a90a-8ea8780af872 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962181dcac283 namespace: kserve-ci-e2e-test resourceVersion: "24926" uid: c321305a-86f4-45d7-a3c5-56dc0873355a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962181e595e93 namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: c42240fc-0677-40a5-9261-0a50215f44aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:37:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:37:00Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962185d18ad63 namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: 7cd96155-771e-4b3f-bf04-ee6d391f8d24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:45:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:37Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a96299016348e6 namespace: kserve-ci-e2e-test resourceVersion: "29281" uid: ff48d731-113a-4d75-8e49-fb364d056420 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: f3f0c258-12af-4ace-a54c-9429480a099a kind: Event lastTimestamp: "2026-04-24T19:45:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:37Z" name: error-404-isvc-11f3c-predictor-5b494597f6-5r9st.18a962990163c806 namespace: kserve-ci-e2e-test resourceVersion: "29282" uid: 08141911-60ca-4e8c-bf49-d715f37eb5c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-11f3c-predictor-5b494597f6 namespace: kserve-ci-e2e-test resourceVersion: "24874" uid: bc07f3a6-473f-4b16-b0f8-524dbfc118a2 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'Created pod: error-404-isvc-11f3c-predictor-5b494597f6-5r9st' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-11f3c-predictor-5b494597f6.18a96217d8a870f7 namespace: kserve-ci-e2e-test resourceVersion: "24885" uid: 43962609-3a6f-41ea-a8b0-081fc0aee473 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24872" uid: 1cdfe5a7-fef7-4d0c-bf90-5d938a954eae kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Scaled up replica set error-404-isvc-11f3c-predictor-5b494597f6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-11f3c-predictor.18a96217d71122ce namespace: kserve-ci-e2e-test resourceVersion: "24876" uid: 6d2cd26e-7c66-4ca0-bfaf-352540424a70 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:36:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24873" uid: 68dc54d4-a317-4705-9616-5d00a3bc67dd kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:52Z" name: error-404-isvc-11f3c-predictor.18a9621b55776d46 namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: 56578f1c-a603-4f75-a4f3-788c4383885c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:36:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24873" uid: 68dc54d4-a317-4705-9616-5d00a3bc67dd kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:52Z" name: error-404-isvc-11f3c-predictor.18a9621b55d89537 namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: a0a65a28-0db2-4013-bf06-1c9f04eede11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:37:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25061" uid: 68dc54d4-a317-4705-9616-5d00a3bc67dd kind: Event lastTimestamp: "2026-04-24T19:37:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:37Z" name: error-404-isvc-11f3c-predictor.18a9622254214639 namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: 55b074ad-11aa-45b1-80c0-511399344e96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:37:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25061" uid: 68dc54d4-a317-4705-9616-5d00a3bc67dd kind: Event lastTimestamp: "2026-04-24T19:37:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:37Z" name: error-404-isvc-11f3c-predictor.18a962225484c686 namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 577ef670-8d48-4173-a723-3fd279cfd700 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "24841" uid: a8f4d4d1-5011-4ca7-b301-43cafd22ff43 kind: Event lastTimestamp: "2026-04-24T19:37:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:37:10Z" name: error-404-isvc-11f3c.18a96217d6e1bd2a namespace: kserve-ci-e2e-test resourceVersion: "25392" uid: 1adeb406-66d5-4469-ae09-763364a5e168 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "24841" uid: a8f4d4d1-5011-4ca7-b301-43cafd22ff43 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-11f3c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-11f3c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-11f3c.18a96217d726df3e namespace: kserve-ci-e2e-test resourceVersion: "24879" uid: e316143a-2951-49b2-ad1e-52baa53326e0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "24841" uid: a8f4d4d1-5011-4ca7-b301-43cafd22ff43 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-11f3c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-11f3c.18a96217d7274129 namespace: kserve-ci-e2e-test resourceVersion: "24881" uid: 418f7e44-c805-4f4f-b98e-4dc5200951ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "25389" uid: a8f4d4d1-5011-4ca7-b301-43cafd22ff43 kind: Event lastTimestamp: "2026-04-24T19:37:10Z" message: InferenceService [error-404-isvc-11f3c] is Ready metadata: creationTimestamp: "2026-04-24T19:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:37:10Z" name: error-404-isvc-11f3c.18a9622317420409 namespace: kserve-ci-e2e-test resourceVersion: "25390" uid: 7955307e-aa4d-4ad4-aba6-0c313bc17b42 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:16.889681Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:16Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cdd38bfca namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: e40dbbdb-eda4-4e32-a11a-7c48c61275e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19105" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cf6c4a448 namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: ede8d076-edd0-43dd-8ea2-5a87f8409a94 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cf833c014 namespace: kserve-ci-e2e-test resourceVersion: "19118" uid: 8186dd7a-70e8-490e-8c3c-68aa0dd679d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cfb4058ed namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: d504824d-7278-4ba1-b5cc-249dbdef7747 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cfbb8eb40 namespace: kserve-ci-e2e-test resourceVersion: "19125" uid: 367942fc-ebf2-4881-8125-de25ae2256fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cfbc4c728 namespace: kserve-ci-e2e-test resourceVersion: "19126" uid: 291a8433-ecbf-4faf-9c97-c5a325a6c4af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cfea17450 namespace: kserve-ci-e2e-test resourceVersion: "19128" uid: 54f6bedf-9fa7-437a-a869-9d9983cd9604 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617cff131b40 namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: f3b69521-d12f-4816-a708-705089a79c50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:25:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:54Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9617d11d9844a namespace: kserve-ci-e2e-test resourceVersion: "19645" uid: b6f1b7ab-b4de-44c5-a61b-5306a4a26dc1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:26:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:26Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9618d2d9c90dd namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 603e7bdc-7dfc-4ab6-b6bd-ed070fcfb9a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:26:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:26Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9618d2d9faf7e namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: 88784023-f1a2-4976-aecf-18055b8f9392 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: cba2074e-1a62-4c76-bbe3-4f68b1110ff0 kind: Event lastTimestamp: "2026-04-24T19:26:29Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:29Z" name: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4.18a9618dd5fc9f80 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: ed2fd227-47c4-4c59-8f90-5e875c3afebb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-443ff-predictor-6f4fd75694 namespace: kserve-ci-e2e-test resourceVersion: "19087" uid: 0ebbadfa-9a5a-459a-b4e0-4a2a5e443555 kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: 'Created pod: error-404-isvc-443ff-predictor-6f4fd75694-x5jf4' metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:16Z" name: error-404-isvc-443ff-predictor-6f4fd75694.18a9617cdcd3a31a namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: 7523fde0-4a32-481e-8a3d-128a03eabc7c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19085" uid: 4c2ece06-86bd-431f-9540-44678a3bffbf kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Scaled up replica set error-404-isvc-443ff-predictor-6f4fd75694 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:16Z" name: error-404-isvc-443ff-predictor.18a9617cdb625403 namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: 16ef5997-890c-4338-be9e-ec36c3e274cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19086" uid: 71584106-37fe-46b5-858e-792a3c69ff37 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:16Z" name: error-404-isvc-443ff-predictor.18a9618059d2af1e namespace: kserve-ci-e2e-test resourceVersion: "19869" uid: 56ee5340-7df3-4a27-b077-584b9ae34f8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19086" uid: 71584106-37fe-46b5-858e-792a3c69ff37 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:16Z" name: error-404-isvc-443ff-predictor.18a961805a326c3d namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: 5f20ff7c-615c-4d31-9a6c-c0a0b7cb0666 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 71584106-37fe-46b5-858e-792a3c69ff37 kind: Event lastTimestamp: "2026-04-24T19:25:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:46Z" name: error-404-isvc-443ff-predictor.18a96183d8d68ef3 namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 34dad566-d354-4bec-9ede-411d3128c154 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 71584106-37fe-46b5-858e-792a3c69ff37 kind: Event lastTimestamp: "2026-04-24T19:25:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:46Z" name: error-404-isvc-443ff-predictor.18a96183d9187a30 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 3711c9ac-2e78-461d-a377-c2e882375ce1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-443ff namespace: kserve-ci-e2e-test resourceVersion: "19067" uid: 4c067cad-ff7b-4c4a-8b9e-dff280df3565 kind: Event lastTimestamp: "2026-04-24T19:26:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:04Z" name: error-404-isvc-443ff.18a9617cdb3948d6 namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: f80d4cca-0240-4bdd-911d-acc7fecd3b95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-443ff namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 4c067cad-ff7b-4c4a-8b9e-dff280df3565 kind: Event lastTimestamp: "2026-04-24T19:26:04Z" message: InferenceService [error-404-isvc-443ff] is Ready metadata: creationTimestamp: "2026-04-24T19:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:04Z" name: error-404-isvc-443ff.18a96188084a89cb namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: aaeabded-8bf5-4032-adde-e657e9a5be10 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:13:52.737625Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13229" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-48e5e-predictor-5fd594f674-4dgzx to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:13:52Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960dd9294d1b7 namespace: kserve-ci-e2e-test resourceVersion: "13234" uid: 59f6fd8a-2e9d-4b4e-beb3-90ff7be028c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13239" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:13:53Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:13:53Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960ddabc96111 namespace: kserve-ci-e2e-test resourceVersion: "13251" uid: 4c03cc5f-c8ac-48cf-a4bc-d8cd919300d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:13:53Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" metadata: creationTimestamp: "2026-04-24T19:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:53Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960ddad43ad3e namespace: kserve-ci-e2e-test resourceVersion: "13253" uid: a39772ce-4fdf-4fee-b46c-5ef71df92418 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" in 15.482s (15.482s including waiting). Image size: 1334717021 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e14819f398 namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: d5cd94b0-db3c-4236-a83c-6ee09edc14d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e14b93768a namespace: kserve-ci-e2e-test resourceVersion: "13433" uid: 6f584774-90a2-4253-bc04-e8b29cff13e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e14c0bca7b namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: c399b065-1eba-4635-adf7-83ea8f019c88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e14c1866b4 namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: 790d4dce-ac88-464b-b9e2-89cc4486809f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:11Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.58s (2.58s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:11Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e1e5e3d164 namespace: kserve-ci-e2e-test resourceVersion: "13458" uid: d31c0997-a1b8-4860-b318-7a5521f2aef6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:11Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e1e9264fa0 namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: 269e70a2-3f8d-4c05-80e1-faf76ef07437 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:11Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e1e9a3692b namespace: kserve-ci-e2e-test resourceVersion: "13462" uid: cbb27ea1-f8f1-4935-a7d3-e6e89ca5b1ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:14:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:47Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960e205bef237 namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: 57c348fd-69b8-485e-988b-5fabcc30a5b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960f25b11b4e0 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 4ac77802-66c1-483f-a0bd-086109c39519 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960f25b123c76 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: b9edd625-6da9-4503-8702-14ee2856b813 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx namespace: kserve-ci-e2e-test resourceVersion: "13231" uid: 457a2da5-4584-4b53-9c42-5fb9d6640940 kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx.18a960f28dc8f7f7 namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 60af8381-40c5-40e1-b3ac-69f86bcaf43b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-48e5e-predictor-5fd594f674 namespace: kserve-ci-e2e-test resourceVersion: "13223" uid: 4794fe32-867c-4d92-9b08-3fd384e62da1 kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: 'Created pod: error-404-isvc-48e5e-predictor-5fd594f674-4dgzx' metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:52Z" name: error-404-isvc-48e5e-predictor-5fd594f674.18a960dd92116c5d namespace: kserve-ci-e2e-test resourceVersion: "13230" uid: 20c94810-e972-46aa-a993-45ecab9d15dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13221" uid: 22264cd9-3660-4ef6-981a-47f4ac90dd9d kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Scaled up replica set error-404-isvc-48e5e-predictor-5fd594f674 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:52Z" name: error-404-isvc-48e5e-predictor.18a960dd8cfabcc8 namespace: kserve-ci-e2e-test resourceVersion: "13225" uid: 09a64324-61ea-4f18-82ea-5294e7d96ab4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:14:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13222" uid: e48af9fe-f0bb-487d-84ca-c5ad4559e6ae kind: Event lastTimestamp: "2026-04-24T19:14:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:22Z" name: error-404-isvc-48e5e-predictor.18a960e10baccfa7 namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: 4e34cc60-33ec-4057-a859-eb5654bc8dc0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:14:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13222" uid: e48af9fe-f0bb-487d-84ca-c5ad4559e6ae kind: Event lastTimestamp: "2026-04-24T19:14:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:22Z" name: error-404-isvc-48e5e-predictor.18a960e10bf9c57e namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: d6d32942-6ae5-4497-9281-9cfe8cbe909a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13426" uid: e48af9fe-f0bb-487d-84ca-c5ad4559e6ae kind: Event lastTimestamp: "2026-04-24T19:15:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:07Z" name: error-404-isvc-48e5e-predictor.18a960e8096ca36b namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: fab86225-9164-4713-9632-2cf7d3ae0332 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13426" uid: e48af9fe-f0bb-487d-84ca-c5ad4559e6ae kind: Event lastTimestamp: "2026-04-24T19:15:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:07Z" name: error-404-isvc-48e5e-predictor.18a960e809b55842 namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 76b17671-d25f-4c5f-a5e4-b0336c0ce0e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-48e5e namespace: kserve-ci-e2e-test resourceVersion: "13194" uid: fc00487d-1f60-44e8-874e-6a5edd9f1203 kind: Event lastTimestamp: "2026-04-24T19:14:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:14:57Z" name: error-404-isvc-48e5e.18a960dd8ccaa78e namespace: kserve-ci-e2e-test resourceVersion: "13891" uid: 18dc86e7-9089-4785-8666-f0dec0a05cca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-48e5e namespace: kserve-ci-e2e-test resourceVersion: "13889" uid: fc00487d-1f60-44e8-874e-6a5edd9f1203 kind: Event lastTimestamp: "2026-04-24T19:14:57Z" message: InferenceService [error-404-isvc-48e5e] is Ready metadata: creationTimestamp: "2026-04-24T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:14:57Z" name: error-404-isvc-48e5e.18a960ecc02ef6c2 namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 234ad651-9541-4276-89aa-75b7a81d48f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:35:12.262501Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23977" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8e956-predictor-579998b9d-qv6bt to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a962077c34f2cf namespace: kserve-ci-e2e-test resourceVersion: "23981" uid: 050727ab-aeb2-4ecd-b7bf-c0fc1534c87e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8e956-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a962078a5a28d4 namespace: kserve-ci-e2e-test resourceVersion: "23991" uid: 24a8f667-52b2-4c26-81cc-5dcecb369b50 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23987" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207b8e6257b namespace: kserve-ci-e2e-test resourceVersion: "24014" uid: 1fbeb836-534c-4e8c-9070-d292e785f1d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207ba26edc2 namespace: kserve-ci-e2e-test resourceVersion: "24016" uid: 4ede7bac-7f48-476e-8fa7-89cc166fd2f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207bd1865e0 namespace: kserve-ci-e2e-test resourceVersion: "24018" uid: 6c50ef35-9fe0-49f3-af08-6f3b90818f34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207bd90d96f namespace: kserve-ci-e2e-test resourceVersion: "24019" uid: ad93a8de-cdcb-41a5-adc7-147cf4d4534a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207bd9c44e5 namespace: kserve-ci-e2e-test resourceVersion: "24020" uid: b0eeb930-fbed-4a75-8a96-a0dc272d04b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207c109eb78 namespace: kserve-ci-e2e-test resourceVersion: "24023" uid: cb024cfd-674c-4f34-91f8-ca8e60da5a64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:13Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207c182aae7 namespace: kserve-ci-e2e-test resourceVersion: "24024" uid: ec973b25-e577-4a97-b017-410d991e55a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:35:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:49Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96207c6978e34 namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: 3e03968e-f591-463f-8fce-a5bc1c685b9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96217d5bddaaa namespace: kserve-ci-e2e-test resourceVersion: "24867" uid: da35a45a-6af0-4683-97fd-4f808728e3b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a96217d5beaf14 namespace: kserve-ci-e2e-test resourceVersion: "24869" uid: a491d54c-7c7d-47d7-9645-e139b9bd7240 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e956-predictor-579998b9d-qv6bt namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: d84ecd56-c38d-41d2-9cc4-a0177cc734a1 kind: Event lastTimestamp: "2026-04-24T19:36:24Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:24Z" name: error-404-isvc-8e956-predictor-579998b9d-qv6bt.18a962184ecb7238 namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: e262115f-29eb-4cb0-aa24-4c36cd4c1d5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8e956-predictor-579998b9d namespace: kserve-ci-e2e-test resourceVersion: "23966" uid: 28d14b4b-88c7-487d-8af7-aeaf2b591c3e kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'Created pod: error-404-isvc-8e956-predictor-579998b9d-qv6bt' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-8e956-predictor-579998b9d.18a962077bd261c5 namespace: kserve-ci-e2e-test resourceVersion: "23978" uid: d49857c4-5e5c-42b9-8ec5-569e783066bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "23963" uid: ec7819ae-89f1-4691-8eae-103e8ce5dfc3 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Scaled up replica set error-404-isvc-8e956-predictor-579998b9d from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-8e956-predictor.18a9620779fe9319 namespace: kserve-ci-e2e-test resourceVersion: "23970" uid: 7b85a104-6cba-48c5-b0b2-aa2cdbb66b15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:35:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "23965" uid: f175d285-8704-4322-b41a-1cc55eac0e32 kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:12Z" name: error-404-isvc-8e956-predictor.18a9620af86dc3cb namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: 50764941-7635-4cc4-931b-404f2e068e22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:35:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "23965" uid: f175d285-8704-4322-b41a-1cc55eac0e32 kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:12Z" name: error-404-isvc-8e956-predictor.18a9620af8e10755 namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: 43ca28bb-9466-4518-878d-c10f68f79134 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "23937" uid: a5282260-3660-4ddf-b4ce-2797a5b80b18 kind: Event lastTimestamp: "2026-04-24T19:35:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:59Z" name: error-404-isvc-8e956.18a9620779d4f345 namespace: kserve-ci-e2e-test resourceVersion: "24574" uid: 949b36fd-302d-44f1-b068-c93227a04178 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "23937" uid: a5282260-3660-4ddf-b4ce-2797a5b80b18 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-8e956": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8e956": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-8e956.18a962077a2eef0d namespace: kserve-ci-e2e-test resourceVersion: "23972" uid: 9b4ab6de-9790-41ac-88b7-edcdd2d074c1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "23937" uid: a5282260-3660-4ddf-b4ce-2797a5b80b18 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8e956": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-8e956.18a962077a2f44ea namespace: kserve-ci-e2e-test resourceVersion: "23974" uid: e6574cbc-567c-42ab-88fc-f9eff5db75b6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "24567" uid: a5282260-3660-4ddf-b4ce-2797a5b80b18 kind: Event lastTimestamp: "2026-04-24T19:35:59Z" message: InferenceService [error-404-isvc-8e956] is Ready metadata: creationTimestamp: "2026-04-24T19:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:59Z" name: error-404-isvc-8e956.18a962128186b95b namespace: kserve-ci-e2e-test resourceVersion: "24568" uid: 33237718-7c8f-4189-b80f-3786532a7d10 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:16:02.021664Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-99560-predictor-59865b4b78-f7h57 to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:16:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:16:02Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbac829fc9 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 52027cd4-5dcc-44e2-98fa-60732a22b170 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-99560-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:02Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbb7e7ab0d namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 2052382c-d4bb-49c7-ab30-527b57209045 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbea08281e namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 2ab9952e-9651-4b25-bc14-e8090f40bd9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbebacf7b4 namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: fdd72376-25ff-4ad6-98cf-adc211ff3dff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbeeabe4c3 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 340b747e-669d-48df-86d8-bb0397063fb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbef283d9f namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: 0d172759-1c39-458c-ae7f-48fdb2dba5d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbef343757 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 50f8cfc9-7440-404e-abf2-d84fa1687c04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbf301962d namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: e6161aea-81b1-48c2-9cfe-4523f126bf23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fbf37ddf17 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 00a0822d-66c4-444f-afdc-e1f22300161a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:16:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:41Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a960fc68e59908 namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 969a4ad8-067b-48a7-91ee-342ace87f792 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:16Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a9617cd3d1c88e namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 33babe02-c5f9-4282-977d-99587ffcb824 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99560-predictor-59865b4b78-f7h57 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 60633c68-5404-4495-8e49-01e91e0f62ba kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:16Z" name: error-404-isvc-99560-predictor-59865b4b78-f7h57.18a9617cd3d434e1 namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 425cdb7b-db1a-430b-918c-ab2cc98ac1e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-99560-predictor-59865b4b78 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 9d0fa4c5-505b-47e4-9e4b-c4fa148a6875 kind: Event lastTimestamp: "2026-04-24T19:16:02Z" message: 'Created pod: error-404-isvc-99560-predictor-59865b4b78-f7h57' metadata: creationTimestamp: "2026-04-24T19:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:02Z" name: error-404-isvc-99560-predictor-59865b4b78.18a960fbac129644 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 86083447-4acc-490c-ab6e-8113119f3f36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-99560-predictor namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: be7c0d5b-6b67-4508-8d23-5fd8bcbc2fb4 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Scaled up replica set error-404-isvc-99560-predictor-59865b4b78 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:01Z" name: error-404-isvc-99560-predictor.18a960fbaaac1448 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 8666c3ca-0bd0-4a0d-8d4f-42b6888d5d60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99560-predictor namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: acee133e-774d-4a40-b551-d13df7dce136 kind: Event lastTimestamp: "2026-04-24T19:17:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:17Z" name: error-404-isvc-99560-predictor.18a960ff291fa28d namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: eae41435-0f35-416d-a628-9ae0c4bd76c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99560-predictor namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: acee133e-774d-4a40-b551-d13df7dce136 kind: Event lastTimestamp: "2026-04-24T19:17:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:17Z" name: error-404-isvc-99560-predictor.18a960ff29825771 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 0b34f675-e949-4619-a62c-014b4ea19fec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99560 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 16896305-d766-4441-879e-f09da4b811b5 kind: Event lastTimestamp: "2026-04-24T19:16:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:51Z" name: error-404-isvc-99560.18a960fbaa748338 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 1a29d04d-daf1-4d7f-a42a-0f079ae1e50b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99560 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 16896305-d766-4441-879e-f09da4b811b5 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-99560": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-99560": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:01Z" name: error-404-isvc-99560.18a960fbaac285fa namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: d28203e9-f276-42ae-88b5-7633aeed5527 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99560 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 16896305-d766-4441-879e-f09da4b811b5 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-99560": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:01Z" name: error-404-isvc-99560.18a960fbaac2d532 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 7ac32bf0-67e8-4a07-9de9-223391bca88c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99560 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 16896305-d766-4441-879e-f09da4b811b5 kind: Event lastTimestamp: "2026-04-24T19:16:51Z" message: InferenceService [error-404-isvc-99560] is Ready metadata: creationTimestamp: "2026-04-24T19:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:51Z" name: error-404-isvc-99560.18a961072381e798 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: c87057dc-0bc8-4928-8f8b-3f019d9b6621 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:35:41.980917Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24347" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620e678ff51c namespace: kserve-ci-e2e-test resourceVersion: "24351" uid: ea37b4b7-32b5-4c62-bc04-3e3619158a82 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a0351-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620e6fe1f715 namespace: kserve-ci-e2e-test resourceVersion: "24361" uid: 6cccf69b-a82d-49f1-aadf-a7fe585093c2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24356" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620ea4880c4e namespace: kserve-ci-e2e-test resourceVersion: "24390" uid: e737b509-ed3c-4e64-8a47-e344bec8aed5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620ea5c128e7 namespace: kserve-ci-e2e-test resourceVersion: "24393" uid: fcd1222e-3ac8-47d1-9f5a-c1422d27a174 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620ea8ec71b7 namespace: kserve-ci-e2e-test resourceVersion: "24395" uid: d94148d4-c0d7-4426-a5a6-784798c95d7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620ea95f669a namespace: kserve-ci-e2e-test resourceVersion: "24396" uid: 16581443-3525-4615-8487-37cf69772816 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620ea96aed0e namespace: kserve-ci-e2e-test resourceVersion: "24397" uid: 424f17a3-8939-4546-bd4c-d2df69c64e15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620eac22b222 namespace: kserve-ci-e2e-test resourceVersion: "24400" uid: 05199458-06e0-4d4d-a779-7ec7003cb129 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:35:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:43Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620eac986c52 namespace: kserve-ci-e2e-test resourceVersion: "24401" uid: d091fd85-6acd-4068-bc8d-d0b10986412f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:36:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:19Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a9620ec8cfd6fe namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 62db4416-c6d9-47dd-9cc1-07651d2fcd09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:53:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:11Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a96302c59d0577 namespace: kserve-ci-e2e-test resourceVersion: "32446" uid: 4e79067e-6956-408f-8713-ead97eea386b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:53:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:11Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a96302c59d9e8b namespace: kserve-ci-e2e-test resourceVersion: "32447" uid: c3d3f0b7-405f-49e1-b3bc-e7e1cc9b5738 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: 76541b04-b9fb-4b4d-b9c6-e9487f3a1b80 kind: Event lastTimestamp: "2026-04-24T19:53:14Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:14Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc.18a963037d7da7c7 namespace: kserve-ci-e2e-test resourceVersion: "32511" uid: c1708d53-f24f-4d2b-adef-7b6528a7c0b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a0351-predictor-5d4c5bbb76 namespace: kserve-ci-e2e-test resourceVersion: "24340" uid: fb7d954a-5f9a-48dc-b5d4-8e4a5ad80511 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: 'Created pod: error-404-isvc-a0351-predictor-5d4c5bbb76-bj4lc' metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-a0351-predictor-5d4c5bbb76.18a9620e670e0154 namespace: kserve-ci-e2e-test resourceVersion: "24348" uid: ec4f4844-cd35-4362-acfd-83b91b882f8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24337" uid: 992d60b9-f0dd-4bcb-a4f5-3b911cbf8be6 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Scaled up replica set error-404-isvc-a0351-predictor-5d4c5bbb76 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-a0351-predictor.18a9620e6588fb75 namespace: kserve-ci-e2e-test resourceVersion: "24342" uid: 79ab0784-4794-4f85-8300-aaef936ed340 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24339" uid: 5e6acd60-51da-4b7f-9b0d-4b233ea162e7 kind: Event lastTimestamp: "2026-04-24T19:35:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:56Z" name: error-404-isvc-a0351-predictor.18a96211e3fd9ced namespace: kserve-ci-e2e-test resourceVersion: "24527" uid: d8c87cea-e7e2-4dee-b3a3-b5c77c2bf5a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24339" uid: 5e6acd60-51da-4b7f-9b0d-4b233ea162e7 kind: Event lastTimestamp: "2026-04-24T19:35:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:56Z" name: error-404-isvc-a0351-predictor.18a96211e4615e89 namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: 5cd9cd53-4f2d-49d7-b237-db97f896bd7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24528" uid: 5e6acd60-51da-4b7f-9b0d-4b233ea162e7 kind: Event lastTimestamp: "2026-04-24T19:37:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:12Z" name: error-404-isvc-a0351-predictor.18a9621562fdf513 namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: c7090acc-08dc-4221-8ded-4660124155c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24528" uid: 5e6acd60-51da-4b7f-9b0d-4b233ea162e7 kind: Event lastTimestamp: "2026-04-24T19:37:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:12Z" name: error-404-isvc-a0351-predictor.18a96215635e8339 namespace: kserve-ci-e2e-test resourceVersion: "25405" uid: 5c265a31-c569-4ab4-a8b9-bf0d186b2038 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 0ad3f071-953b-40f7-8efe-bd606a7b7da0 kind: Event lastTimestamp: "2026-04-24T19:36:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:29Z" name: error-404-isvc-a0351.18a9620e65677b28 namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: e5261582-cfbf-45d1-a2d4-04d86b590129 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 0ad3f071-953b-40f7-8efe-bd606a7b7da0 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-a0351": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a0351": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-a0351.18a9620e65aac42e namespace: kserve-ci-e2e-test resourceVersion: "24344" uid: 022b0f5e-256c-4701-8649-5d2b9134bd88 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 0ad3f071-953b-40f7-8efe-bd606a7b7da0 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a0351": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-a0351.18a9620e65ab400a namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: f9bcc204-76a3-48a6-91f0-5d7707b92a6c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24997" uid: 0ad3f071-953b-40f7-8efe-bd606a7b7da0 kind: Event lastTimestamp: "2026-04-24T19:36:29Z" message: InferenceService [error-404-isvc-a0351] is Ready metadata: creationTimestamp: "2026-04-24T19:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:29Z" name: error-404-isvc-a0351.18a96219835d0553 namespace: kserve-ci-e2e-test resourceVersion: "24998" uid: 11d7733a-68ba-4654-ab7a-cd3a97371cdb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:24:36.997421Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a9617393751d0a namespace: kserve-ci-e2e-test resourceVersion: "18653" uid: b0d540ca-ad40-4985-95cd-29ba1281af38 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18661" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173ad8d332a namespace: kserve-ci-e2e-test resourceVersion: "18676" uid: 86d24395-7c49-48c9-bc0b-96d754c0a6e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173aecea6ff namespace: kserve-ci-e2e-test resourceVersion: "18678" uid: b85264b7-4676-4f1c-a500-790c99f39f37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173b1cfd510 namespace: kserve-ci-e2e-test resourceVersion: "18682" uid: 56ce80ff-9ecb-4e21-84ed-c00216eb5fb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173b24ab9b4 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 14465546-2f8d-4100-bd32-d0fe9a155578 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173b25785ce namespace: kserve-ci-e2e-test resourceVersion: "18684" uid: 060943a0-97ab-4ea8-93fd-cd35bd4c4077 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173b4a7e212 namespace: kserve-ci-e2e-test resourceVersion: "18687" uid: bf6aeee0-56f6-4bd3-89ca-c17c0ea5f054 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173b5169b69 namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 55c99376-3fda-4829-ad7e-a1fc23f87865 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:25:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:13Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96173b98fb656 namespace: kserve-ci-e2e-test resourceVersion: "18977" uid: b0c0b9d3-36d5-4e86-80c3-7739336628bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96183f3d603b1 namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: 76f4043d-42ec-4649-9fce-eac41275c69a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a96183f3d65c11 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: 98bdbc81-df08-49c4-ab2e-9b8ac5d57ea6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c namespace: kserve-ci-e2e-test resourceVersion: "18651" uid: 0ddbb448-03b5-46b3-9277-eae73d6103ba kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c.18a9618441d8ced2 namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: c3ed50a6-a1ca-4092-8a74-310cb2b17a8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2e49-predictor-5cc74f54d8 namespace: kserve-ci-e2e-test resourceVersion: "18639" uid: a66397b4-1ee3-400e-995c-7c9e61f740e4 kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: 'Created pod: error-404-isvc-b2e49-predictor-5cc74f54d8-s2p4c' metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-b2e49-predictor-5cc74f54d8.18a96173930e0836 namespace: kserve-ci-e2e-test resourceVersion: "18650" uid: 7fba070c-b420-4667-b3fe-7a36e0112464 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: 205b1649-82b2-4495-b6c0-139a36393b92 kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Scaled up replica set error-404-isvc-b2e49-predictor-5cc74f54d8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-b2e49-predictor.18a961739170fdec namespace: kserve-ci-e2e-test resourceVersion: "18643" uid: 718ab459-5ad8-4ced-ae33-8edb5f14308d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18638" uid: 0f652904-8625-40e8-8500-02f1aa4c8c8c kind: Event lastTimestamp: "2026-04-24T19:24:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:51Z" name: error-404-isvc-b2e49-predictor.18a96177100d09d1 namespace: kserve-ci-e2e-test resourceVersion: "18814" uid: 2e3e0c77-632c-4f85-8712-aa5f65b28cca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18638" uid: 0f652904-8625-40e8-8500-02f1aa4c8c8c kind: Event lastTimestamp: "2026-04-24T19:24:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:51Z" name: error-404-isvc-b2e49-predictor.18a96177106a3ef7 namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: adaf1828-a706-4b9c-834c-29d1cb07cda4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: 0f652904-8625-40e8-8500-02f1aa4c8c8c kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:37Z" name: error-404-isvc-b2e49-predictor.18a9617a8f1c11a9 namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: a9c88f50-54e2-43d1-b41a-3f0260e898c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18815" uid: 0f652904-8625-40e8-8500-02f1aa4c8c8c kind: Event lastTimestamp: "2026-04-24T19:25:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:37Z" name: error-404-isvc-b2e49-predictor.18a9617a8f811bd8 namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: b5a9d82e-01a5-4763-a827-abf1ea0c9c4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "18609" uid: 64c6fd17-bb34-4605-90b6-80660fc0b44c kind: Event lastTimestamp: "2026-04-24T19:25:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:23Z" name: error-404-isvc-b2e49.18a961739135cee0 namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: 9254e3de-296a-4081-a473-cc5171d63045 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "18609" uid: 64c6fd17-bb34-4605-90b6-80660fc0b44c kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-b2e49": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2e49": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-b2e49.18a9617391cf7d32 namespace: kserve-ci-e2e-test resourceVersion: "18645" uid: 38c9c092-d6ca-4d45-808f-c6572b6af1a6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "18609" uid: 64c6fd17-bb34-4605-90b6-80660fc0b44c kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2e49": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-b2e49.18a9617391cfbf19 namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: 5deae3ac-39e3-4523-b2db-b85f3fed5cb0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "19205" uid: 64c6fd17-bb34-4605-90b6-80660fc0b44c kind: Event lastTimestamp: "2026-04-24T19:25:23Z" message: InferenceService [error-404-isvc-b2e49] is Ready metadata: creationTimestamp: "2026-04-24T19:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:23Z" name: error-404-isvc-b2e49.18a9617e73df09ef namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: c8df649b-dfbd-41f9-a48f-fb5a31806c58 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:26:27.123258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d377803ef namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 953399e6-bcaa-467c-88ed-f57bbb495b59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20069" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d506c5133 namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 310c347d-e744-4a8d-b09b-57ee58514953 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d51a2137f namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: 1efae1ec-5aed-4766-a741-fb256096e035 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d54cc8999 namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: 7f2ce201-69b7-4cb3-ad8b-0d2a2c1ad29c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d553ceaf1 namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 410439ef-44aa-455d-ae68-20c04e75a292 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d55485ff9 namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: 867ce17c-3d63-4423-a299-19c1ecd7350f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d586cbee8 namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 538d7eed-710c-46f0-8a12-0d37c7edecec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618d58e510b4 namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: f5a4b511-9bd1-4643-a732-62920f42de8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:26:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:35:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:45Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9618da84c935f namespace: kserve-ci-e2e-test resourceVersion: "24424" uid: d6f1f2d7-f3c1-421c-9a1d-5df2ae4add12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9620e64798412 namespace: kserve-ci-e2e-test resourceVersion: "24334" uid: 1a55f090-09f9-4081-a7fa-f405849c7dc7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:41Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9620e647a070a namespace: kserve-ci-e2e-test resourceVersion: "24338" uid: 74a5736d-5090-4795-9c74-d858f211dc7e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fcf9162-136e-4823-b6a2-285c935c7f14 kind: Event lastTimestamp: "2026-04-24T19:35:45Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:45Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7.18a9620f1caeada3 namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 892127cf-f902-4873-9ca4-dc4780f06ff3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cb3b2-predictor-754d94dcdd namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: e939a54a-4dbe-46bd-966f-f1739349bd83 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'Created pod: error-404-isvc-cb3b2-predictor-754d94dcdd-qgrb7' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor-754d94dcdd.18a9618d36f55529 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 1abedbde-19ce-46ab-8bd5-81724822f356 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: 1f59cfe1-565d-41f3-957f-941aec16fe04 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Scaled up replica set error-404-isvc-cb3b2-predictor-754d94dcdd from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2-predictor.18a9618d35c7207e namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 9d9cb116-b592-4f28-b3d6-49910a78e0c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: c6ef97eb-9201-45d5-8bf5-c7993871eceb kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:57Z" name: error-404-isvc-cb3b2-predictor.18a96190b4004b54 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 44295e09-722a-488d-bea6-3f7edecb5d2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: c6ef97eb-9201-45d5-8bf5-c7993871eceb kind: Event lastTimestamp: "2026-04-24T19:26:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:57Z" name: error-404-isvc-cb3b2-predictor.18a96190b4697e15 namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: e120c3bc-66db-48b8-aa22-8dc58abcf560 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: c6ef97eb-9201-45d5-8bf5-c7993871eceb kind: Event lastTimestamp: "2026-04-24T19:27:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:42Z" name: error-404-isvc-cb3b2-predictor.18a96197b1e9b157 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: c73ffa97-709a-4bba-8452-6a32e561b5ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: c6ef97eb-9201-45d5-8bf5-c7993871eceb kind: Event lastTimestamp: "2026-04-24T19:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:42Z" name: error-404-isvc-cb3b2-predictor.18a96197b24fa330 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 65df6127-532d-4cc3-9646-eef217e706c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 29fc44bf-ff0b-455c-b3ba-24f9a9596b56 kind: Event lastTimestamp: "2026-04-24T19:27:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:15Z" name: error-404-isvc-cb3b2.18a9618d354e1356 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 9826b417-b057-4e2a-ae41-73174c685aae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 29fc44bf-ff0b-455c-b3ba-24f9a9596b56 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-cb3b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cb3b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2.18a9618d358fc45b namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: a81d9af4-4933-4e78-b3f0-7ce95137ccdd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 29fc44bf-ff0b-455c-b3ba-24f9a9596b56 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cb3b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:27Z" name: error-404-isvc-cb3b2.18a9618d3590280e namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: c105017e-1cd0-4dd4-a15a-0d675e3727d6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: 29fc44bf-ff0b-455c-b3ba-24f9a9596b56 kind: Event lastTimestamp: "2026-04-24T19:27:15Z" message: InferenceService [error-404-isvc-cb3b2] is Ready metadata: creationTimestamp: "2026-04-24T19:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:15Z" name: error-404-isvc-cb3b2.18a9619862b275f7 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: a989db29-4e5d-4275-b95d-ec89f4adf0a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:15:22.115906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d5820-predictor-5b7d74d57d-j44tx to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f261f11816 namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 6db012e9-6b89-45ea-bb04-b8b2be6f6c2b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d5820-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f273d4b6c5 namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 86f3caf5-394a-4283-9c77-e60c59f70261 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f29f42cebc namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 16e1f23b-7027-4548-8864-0e52e596f8e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2a09cccb1 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 3c4fdf13-ac71-44bc-99dd-a4a471782183 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2a32f1af6 namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 01dcbb97-4e95-4c14-b019-ba3a7e9ee056 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2a3a81404 namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 39475774-7fb9-45b5-ac2e-82163cf2d1e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2a3b52056 namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 24fbb6fd-131c-44ac-b277-4ceab7849f79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2a66ddb26 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 92444f7f-9720-4e3e-bc7a-e22604bafad6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2a708b1d8 namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: a684ac4c-e840-45f2-809d-7f7e438f0d0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:24:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:40Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a960f2d561aa93 namespace: kserve-ci-e2e-test resourceVersion: "18721" uid: 53acca18-7465-4860-87f8-5976eb7d1823 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a961738f58207c namespace: kserve-ci-e2e-test resourceVersion: "18622" uid: 7d4987e4-6e94-46a7-b3dd-1fb31ea3fb16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:36Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a961738f58b176 namespace: kserve-ci-e2e-test resourceVersion: "18625" uid: a9c673a4-4feb-46e0-a966-63a5ce9743ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 3a8cdfef-d943-4b8b-8641-ef57122233aa kind: Event lastTimestamp: "2026-04-24T19:24:40Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:40Z" name: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx.18a9617449b3f0cd namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: c3f91dfe-c1d4-4544-ab2b-c3bc01ed47da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d5820-predictor-5b7d74d57d namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: dde17540-f5e9-49dd-a659-f1d71c3b8398 kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'Created pod: error-404-isvc-d5820-predictor-5b7d74d57d-j44tx' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-d5820-predictor-5b7d74d57d.18a960f2615c2596 namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 62b2e5fd-8134-475f-bdf4-fb898d5c17c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 17d72d50-e74a-4804-9d3f-2010e8a9e325 kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: Scaled up replica set error-404-isvc-d5820-predictor-5b7d74d57d from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-d5820-predictor.18a960f25fdfbf0b namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 08509a27-3089-448d-8b2a-cc7470d03380 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:15:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: 5748054a-5e5d-41ef-b6c0-ab380c8d9ae4 kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:15:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:52Z" name: error-404-isvc-d5820-predictor.18a960f5de608bbb namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 1321f5a2-5124-4e96-a9d8-fff991ed17ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:15:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: 5748054a-5e5d-41ef-b6c0-ab380c8d9ae4 kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:15:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:52Z" name: error-404-isvc-d5820-predictor.18a960f5dec52f8d namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: 07639ab1-398f-4566-8a03-b4baf02f14c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:16:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 5748054a-5e5d-41ef-b6c0-ab380c8d9ae4 kind: Event lastTimestamp: "2026-04-24T19:16:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:52Z" name: error-404-isvc-d5820-predictor.18a960fcdc2b62b7 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 39075316-bc3e-4ebb-b409-c7987f8dcd5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:16:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 5748054a-5e5d-41ef-b6c0-ab380c8d9ae4 kind: Event lastTimestamp: "2026-04-24T19:16:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:52Z" name: error-404-isvc-d5820-predictor.18a960fcdc90ffa5 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: e65511e6-0355-4d0f-af4d-914424083e2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 354801a0-72a2-48a0-a2d6-726da7441acc kind: Event lastTimestamp: "2026-04-24T19:16:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:10Z" name: error-404-isvc-d5820.18a960f25fb28feb namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 85efb664-c4e7-4046-afeb-23e25f3e728a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 354801a0-72a2-48a0-a2d6-726da7441acc kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-d5820": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d5820": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-d5820.18a960f26010a690 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 9ea57e45-05c8-4227-aec7-c7d510edf25b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 354801a0-72a2-48a0-a2d6-726da7441acc kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d5820": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:22Z" name: error-404-isvc-d5820.18a960f26010f183 namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: fe6e4066-8d44-46ba-b2ed-e68840d56633 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 354801a0-72a2-48a0-a2d6-726da7441acc kind: Event lastTimestamp: "2026-04-24T19:16:10Z" message: InferenceService [error-404-isvc-d5820] is Ready metadata: creationTimestamp: "2026-04-24T19:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:10Z" name: error-404-isvc-d5820.18a960fd9010bb38 namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: fd6313bc-85f7-4985-8e7c-ff99dac969c7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:47.386153Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ec538-predictor-5cc86fcf96-24dwx to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a96183f6f3f6b6 namespace: kserve-ci-e2e-test resourceVersion: "19545" uid: f49acaa4-aaa4-4247-b046-d7c8bae7c79f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a9618410850775 namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: d9644270-9b6b-4f65-8503-044f2adb177d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a9618411cd1103 namespace: kserve-ci-e2e-test resourceVersion: "19567" uid: ca6e6d21-472e-4007-ba0e-4f7630e7b533 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a9618414fd2ce7 namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: c9637c03-8337-44c8-bea2-3b7cae0bb5e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a9618415711fc1 namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: a313a321-5b15-421b-bb91-16cec464782d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a96184157de5ea namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: e4920d36-94a9-4a8f-942f-2b2d3ac28668 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a9618419d427e5 namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: 64be6fd2-14aa-4424-84e3-89fffef8d69d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a961841a525678 namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: b52d30e6-c6ca-4d87-ab70-4156e75aefb2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:25:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:26:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:25Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a961848bef67a0 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 6513d752-20fd-4489-ae0a-439dd6a3c727 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a962077859807f namespace: kserve-ci-e2e-test resourceVersion: "23954" uid: 1449fe93-4b16-4f60-936a-a8ae0f41bd6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: a6f61d88-8d78-4f16-bc8b-ec61b385dcf6 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx.18a962077859dbf4 namespace: kserve-ci-e2e-test resourceVersion: "23957" uid: 698b2b53-87df-4b65-8537-5ed8ade8b828 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ec538-predictor-5cc86fcf96 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 5b752ba0-3971-4eae-98c0-4f86a6d79482 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Created pod: error-404-isvc-ec538-predictor-5cc86fcf96-24dwx' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor-5cc86fcf96.18a96183f67f4e4b namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: d64efe1e-8d15-4440-9b7c-3a905ab7659d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: d7c3efd8-30c8-48c4-ba63-b1d81da46680 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Scaled up replica set error-404-isvc-ec538-predictor-5cc86fcf96 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538-predictor.18a96183f4de9feb namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: ed25b8fa-f949-4ebe-99ff-6e9f160f6ff5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: cb21c9bf-6cdc-44a4-ba26-cec45e0160d7 kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:17Z" name: error-404-isvc-ec538-predictor.18a96187735640d7 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: a97c81e2-61e9-42b2-9eff-17afac993c17 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: cb21c9bf-6cdc-44a4-ba26-cec45e0160d7 kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:17Z" name: error-404-isvc-ec538-predictor.18a9618773b99274 namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: e2f489a9-3751-4ce3-8f8a-672c67e61a55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: cb21c9bf-6cdc-44a4-ba26-cec45e0160d7 kind: Event lastTimestamp: "2026-04-24T19:26:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:47Z" name: error-404-isvc-ec538-predictor.18a9618e7120b3ce namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: b2ba89f0-3a25-4ee7-bdf8-0304af9b60e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: cb21c9bf-6cdc-44a4-ba26-cec45e0160d7 kind: Event lastTimestamp: "2026-04-24T19:26:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:47Z" name: error-404-isvc-ec538-predictor.18a9618e7180c93f namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 109da858-00dc-4eef-b791-534df9789f47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "19501" uid: 5dde9012-087a-4510-99fe-9d43f007c39a kind: Event lastTimestamp: "2026-04-24T19:26:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:35Z" name: error-404-isvc-ec538.18a96183f4ad09a0 namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: cabd6277-c496-4cf4-8cae-9f97263b8cb5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "19501" uid: 5dde9012-087a-4510-99fe-9d43f007c39a kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-ec538": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ec538": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538.18a96183f512afda namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: eff3fbeb-63dd-4079-b9e8-d6b6e7618d4c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "19501" uid: 5dde9012-087a-4510-99fe-9d43f007c39a kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ec538": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:47Z" name: error-404-isvc-ec538.18a96183f512fe64 namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: c882ea93-69cd-4b4d-ba5f-ea3ab6134990 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 5dde9012-087a-4510-99fe-9d43f007c39a kind: Event lastTimestamp: "2026-04-24T19:26:35Z" message: InferenceService [error-404-isvc-ec538] is Ready metadata: creationTimestamp: "2026-04-24T19:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:35Z" name: error-404-isvc-ec538.18a9618f4621e4e9 namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 08793b46-6e0b-48c6-abda-1f79616cc985 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:13:51.769085Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13110" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp to ip-10-0-129-151.ec2.internal metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:13:51Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960dd58da0290 namespace: kserve-ci-e2e-test resourceVersion: "13116" uid: 0ebbb1a4-eb98-4190-905d-681683dfec85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13122" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960dd746c5d5e namespace: kserve-ci-e2e-test resourceVersion: "13196" uid: 21b7dfc0-8370-4b35-b1f2-f33ed427128c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960dd75eb1f08 namespace: kserve-ci-e2e-test resourceVersion: "13198" uid: f8992131-d447-42d1-af4c-824fc6631815 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" in 4.306s (4.306s including waiting). Image size: 301492123 bytes.' metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960de769ad51f namespace: kserve-ci-e2e-test resourceVersion: "13293" uid: 1de6ac98-b19b-42ee-b60e-43d69fc3e94d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960de79f0e877 namespace: kserve-ci-e2e-test resourceVersion: "13298" uid: f1869c08-18a8-4c3b-b792-ffaf28630330 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960de7a6730d3 namespace: kserve-ci-e2e-test resourceVersion: "13299" uid: f91df09c-92c0-47c3-8ef7-0c901f7308e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:01Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" metadata: creationTimestamp: "2026-04-24T19:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:01Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960df8b7b9a25 namespace: kserve-ci-e2e-test resourceVersion: "13360" uid: b0e39e07-6c2f-4b1d-801f-828f3a5d9c1e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:14Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" in 13.279s (13.279s including waiting). Image size: 1560924583 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:14Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e2a307b626 namespace: kserve-ci-e2e-test resourceVersion: "13500" uid: bf4ee2f2-a8bb-438a-91d0-7124260e856c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:14Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e2a643b4c7 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: d334df2e-f386-422b-a72c-4ea2f7d15391 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:14Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e2a6c91cb0 namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: e85ff58f-2761-4b36-8070-9ea2f07fa02b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:14Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T19:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:14Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e2a6d5daae namespace: kserve-ci-e2e-test resourceVersion: "13503" uid: cd216ed9-4a31-4010-b402-cdd05c8bc4fb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.243s (2.243s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:16Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e32c8d83fe namespace: kserve-ci-e2e-test resourceVersion: "13520" uid: d3b0da95-600c-42cf-a9f0-0ca84fbb7dc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:16Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e330904594 namespace: kserve-ci-e2e-test resourceVersion: "13523" uid: 2e32456c-0960-4099-9b89-66c2360b5469 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:14:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:16Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e3310a3ac5 namespace: kserve-ci-e2e-test resourceVersion: "13524" uid: a8732ea0-4372-48e8-9f47-0e98dc922490 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:16:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:04Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960e38572dacc namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 1fa53f59-f09d-429e-8e45-2aae7449ab6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960fba908e9a5 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: 20a62bb6-25c0-4e75-8b78-cfb58818899a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960fba9097f14 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 8a2d83b5-0f1c-4883-b26d-d339b28a0654 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp namespace: kserve-ci-e2e-test resourceVersion: "13113" uid: 192bef1a-d5ab-4b61-8bef-cf58bf10358d kind: Event lastTimestamp: "2026-04-24T19:16:04Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:04Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp.18a960fc33b34ff0 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 88809769-1580-43f3-8037-26b524a36afe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-151.ec2.internal source: component: kubelet host: ip-10-0-129-151.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7cb986cd7 namespace: kserve-ci-e2e-test resourceVersion: "13099" uid: 21a010e3-de07-4728-b6c0-d30cdf93115d kind: Event lastTimestamp: "2026-04-24T19:13:51Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7cb986cd7-plzkp' metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:51Z" name: isvc-sklearn-graph-1-predictor-7cb986cd7.18a960dd58737a4b namespace: kserve-ci-e2e-test resourceVersion: "13112" uid: bdbc3fc5-5d57-48c9-a945-2350f5b5f38b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13097" uid: 9e39deb8-cbf4-4c29-b614-c62e5f3ba4ba kind: Event lastTimestamp: "2026-04-24T19:13:51Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7cb986cd7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:51Z" name: isvc-sklearn-graph-1-predictor.18a960dd51c235a5 namespace: kserve-ci-e2e-test resourceVersion: "13101" uid: 28bae4f2-9e5b-4388-bda6-52cc00fe0857 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13106" uid: c5744408-8d4b-4771-a27d-6887f38d408c kind: Event lastTimestamp: "2026-04-24T19:14:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:51Z" name: isvc-sklearn-graph-1-predictor.18a960e0d72e47fe namespace: kserve-ci-e2e-test resourceVersion: "13824" uid: dc6971a5-0a1e-497d-8968-7d1d76a933cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13106" uid: c5744408-8d4b-4771-a27d-6887f38d408c kind: Event lastTimestamp: "2026-04-24T19:14:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:51Z" name: isvc-sklearn-graph-1-predictor.18a960e0d796e3c5 namespace: kserve-ci-e2e-test resourceVersion: "13825" uid: b7d6b090-30ba-4393-a308-fd73ffcc9cf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:14:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: c5744408-8d4b-4771-a27d-6887f38d408c kind: Event lastTimestamp: "2026-04-24T19:15:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:51Z" name: isvc-sklearn-graph-1-predictor.18a960e7d557bccd namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 3c47183d-1152-4dbe-8fc9-039082dfde3c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:14:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: c5744408-8d4b-4771-a27d-6887f38d408c kind: Event lastTimestamp: "2026-04-24T19:15:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:51Z" name: isvc-sklearn-graph-1-predictor.18a960e7d5b7ed3c namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: a2c007bd-4623-4e7b-95f0-f342bdc69b0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13081" uid: 41ece607-ebea-4cda-9a7a-10fe0606915c kind: Event lastTimestamp: "2026-04-24T19:15:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:24Z" name: isvc-sklearn-graph-1.18a960dd57cc0795 namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: 1c5439ee-be1e-43aa-b2d9-00dc0a00d570 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 41ece607-ebea-4cda-9a7a-10fe0606915c kind: Event lastTimestamp: "2026-04-24T19:15:24Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-24T19:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:24Z" name: isvc-sklearn-graph-1.18a960f2e7cb66da namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 331828ba-f802-4801-9fe8-64c0f603d3a4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:13:52.427875Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13202" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960dd801e5b05 namespace: kserve-ci-e2e-test resourceVersion: "13207" uid: f88170aa-30ab-45bf-b0e8-a80e5147e258 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13212" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960dd99b2c409 namespace: kserve-ci-e2e-test resourceVersion: "13242" uid: 29233db9-243d-48e6-97a6-7b979f8d4080 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960dd9b04c046 namespace: kserve-ci-e2e-test resourceVersion: "13244" uid: 0dc2b16b-7ecf-46f7-917d-3c6e5c7aac2f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" in 3.648s (3.649s including waiting). Image size: 301492123 bytes.' metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960de74848a61 namespace: kserve-ci-e2e-test resourceVersion: "13291" uid: 8d66c2e9-d1a2-4f63-bfc4-119793b77f72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960de78b02bb9 namespace: kserve-ci-e2e-test resourceVersion: "13294" uid: a3c491c8-130a-4d55-8ef9-183aabf00ae0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960de791fedf9 namespace: kserve-ci-e2e-test resourceVersion: "13296" uid: 3921c4f6-888f-4c3c-b245-c012e9c276d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:00Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" metadata: creationTimestamp: "2026-04-24T19:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:00Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960df6a7a54f9 namespace: kserve-ci-e2e-test resourceVersion: "13346" uid: abb1b384-0951-40bc-b12e-17464c9e85cc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" in 24s (24s including waiting). Image size: 1560924583 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e50108281f namespace: kserve-ci-e2e-test resourceVersion: "13592" uid: bbbf8bd9-7948-4b1b-b707-440f0956dfcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e50413ecfc namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: 9769a4dd-c1e6-4446-985c-68823c2600fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e5048911d2 namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: 5e5ec6b9-daad-457a-bcd9-526ce1cc7d6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e504949609 namespace: kserve-ci-e2e-test resourceVersion: "13595" uid: 6d15a1f7-655e-4c69-8961-d072a6a35bbd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:27Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.378s (2.378s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:27Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e59256d6d3 namespace: kserve-ci-e2e-test resourceVersion: "13614" uid: 8dcf173a-91d8-43c2-8867-f4ebfcb5e7e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:27Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e5960387cb namespace: kserve-ci-e2e-test resourceVersion: "13616" uid: 212fff20-0337-40e2-ac8e-8ad9bc8e42bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:14:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:27Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e59687f86c namespace: kserve-ci-e2e-test resourceVersion: "13618" uid: 5f296e1a-e374-4a35-b92d-876350328e00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:14:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:16:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:04Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960e5f6dab20f namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 88ceb89d-61b3-4044-b554-82667584585a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960fba58e2adf namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 4db97151-8307-4dbc-bd1c-06354f25827e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960fba58ea2de namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 95a29e4d-f413-4b5d-bcd7-09656af63697 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv namespace: kserve-ci-e2e-test resourceVersion: "13205" uid: 4c4be8d4-5f52-4c1c-9ad0-de17127af99a kind: Event lastTimestamp: "2026-04-24T19:16:04Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:04Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv.18a960fc510a8a68 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 65fc681c-1ce0-4d80-b165-e3eb483aa0cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-8c9c45b69 namespace: kserve-ci-e2e-test resourceVersion: "13184" uid: 86592d9d-f269-463f-bafa-47498580f8d3 kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-8c9c45b69-nq2zv' metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-2-predictor-8c9c45b69.18a960dd7fc0eafb namespace: kserve-ci-e2e-test resourceVersion: "13204" uid: 48fdf15f-6766-42fd-8426-4a0e9650cb40 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13180" uid: 3bf3e449-2147-45d8-8eb4-ccc9d67cb91a kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-8c9c45b69 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:52Z" name: isvc-sklearn-graph-2-predictor.18a960dd6960b758 namespace: kserve-ci-e2e-test resourceVersion: "13186" uid: 2d8211f3-5fe3-4b5e-96d5-3c7691f29062 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13182" uid: 66109e3d-2730-4c3f-90a7-51e44847f550 kind: Event lastTimestamp: "2026-04-24T19:14:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:37Z" name: isvc-sklearn-graph-2-predictor.18a960e0e7e11573 namespace: kserve-ci-e2e-test resourceVersion: "13708" uid: e7ede752-67c8-4308-9be4-31123982cea5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13182" uid: 66109e3d-2730-4c3f-90a7-51e44847f550 kind: Event lastTimestamp: "2026-04-24T19:14:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:37Z" name: isvc-sklearn-graph-2-predictor.18a960e0e84567fc namespace: kserve-ci-e2e-test resourceVersion: "13709" uid: 1980ea1c-200e-4e9d-a169-eb157f215e2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:14:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13418" uid: 66109e3d-2730-4c3f-90a7-51e44847f550 kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:52Z" name: isvc-sklearn-graph-2-predictor.18a960eb64f4f3e7 namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: f838ee16-d089-4483-b72e-9d60aa7ade3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:14:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13418" uid: 66109e3d-2730-4c3f-90a7-51e44847f550 kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:52Z" name: isvc-sklearn-graph-2-predictor.18a960eb65c74bfd namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: b6e87504-4239-43e4-972f-ecc964f877f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13162" uid: 05a4769f-358f-4973-abf6-eff9154a7b53 kind: Event lastTimestamp: "2026-04-24T19:15:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:34Z" name: isvc-sklearn-graph-2.18a960dd69337ff3 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 3d704d34-ffee-48c4-bdb9-af01af2f6e86 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 05a4769f-358f-4973-abf6-eff9154a7b53 kind: Event lastTimestamp: "2026-04-24T19:15:34Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-24T19:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:34Z" name: isvc-sklearn-graph-2.18a960f5596403e1 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: fec59b8f-7c22-44cb-891a-438f6fa06e32 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:13:51.972129Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13163" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-nnbc7 to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:13:51Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960dd64f43913 namespace: kserve-ci-e2e-test resourceVersion: "13167" uid: 09bb8e8a-d77e-4dcc-ae64-d6330ba15f1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13171" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:13:52Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960dd7dbcd7a8 namespace: kserve-ci-e2e-test resourceVersion: "13199" uid: 85867ccf-df28-4b3d-bc6d-190da712a4cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:52Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960dd7f2b373c namespace: kserve-ci-e2e-test resourceVersion: "13201" uid: 0f466f2a-c582-48c2-8a47-768a1d16a4be reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" in 4.126s (4.126s including waiting). Image size: 301492123 bytes.' metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960de7525050b namespace: kserve-ci-e2e-test resourceVersion: "13292" uid: 72039ccc-6fee-4ae4-ba03-36bc5c1da249 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960de78bb9d02 namespace: kserve-ci-e2e-test resourceVersion: "13295" uid: 198e3304-4666-4dc4-b6d6-5c3fbf0410a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:13:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960de792d10d6 namespace: kserve-ci-e2e-test resourceVersion: "13297" uid: 9890129d-4825-44a8-a5ed-899281fbdccf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:00Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T19:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960df6a8f6134 namespace: kserve-ci-e2e-test resourceVersion: "13347" uid: 74f451bd-9e6a-44ff-8b4f-7dc671aa340d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.112s (24.112s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e507c56c8b namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: f23e9cab-612b-4d7f-90d9-fadcf1fe1c67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e50ae88d69 namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: be9c320f-a2c6-4310-aa6d-0130d54b6dce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e50b55b900 namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 1846c9c7-626f-4a86-9014-44fcecf7b4c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:24Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T19:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:24Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e50b60a462 namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: 91a11caf-9f0b-41e4-b229-fee453d44aca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:27Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.27s (2.27s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e592ba629d namespace: kserve-ci-e2e-test resourceVersion: "13615" uid: 234eab89-6e74-48c5-b80c-1ff082cd7578 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e59614e3b1 namespace: kserve-ci-e2e-test resourceVersion: "13617" uid: 42790f62-ade4-4783-92f3-a59ce283b39f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:14:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e59696efb4 namespace: kserve-ci-e2e-test resourceVersion: "13619" uid: 4ca85741-d1a4-4c5d-b2ac-90244e9203b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T19:14:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960e5bb10029d namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 53516c85-2f51-451e-a60e-89cf0269a1ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960fba257660f namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: db9631c3-595b-4656-a7d7-16cda9e59d7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960fba25c1450 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: af5df795-fdfc-4bd0-a13c-4b1015242e1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7 namespace: kserve-ci-e2e-test resourceVersion: "13165" uid: bed5ebf8-3246-41fb-acca-4d84f4c2a5d3 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-nnbc7.18a960fc153bd589 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 9a8f073c-5a70-4ce5-a239-b2db9a711832 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13156" uid: 69368dc4-ffc4-49b2-b277-152bb28fdacf kind: Event lastTimestamp: "2026-04-24T19:13:51Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-nnbc7' metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:51Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a960dd648dc5d2 namespace: kserve-ci-e2e-test resourceVersion: "13164" uid: 0863565d-8b9b-4c22-bd1e-d42b7106c7d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13154" uid: 2c08329c-7985-417c-a223-065ab6feb188 kind: Event lastTimestamp: "2026-04-24T19:13:51Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:51Z" name: isvc-xgboost-graph-predictor.18a960dd622dc955 namespace: kserve-ci-e2e-test resourceVersion: "13159" uid: 4394b7e4-0ece-4b33-a308-72f862cf8c78 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13155" uid: 8963827d-bb6d-4a81-9aaf-5ccb73206ac4 kind: Event lastTimestamp: "2026-04-24T19:15:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:06Z" name: isvc-xgboost-graph-predictor.18a960e0e0b65a00 namespace: kserve-ci-e2e-test resourceVersion: "13971" uid: 3a48261e-7393-4e75-82a8-860b81cbb69f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13155" uid: 8963827d-bb6d-4a81-9aaf-5ccb73206ac4 kind: Event lastTimestamp: "2026-04-24T19:15:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:06Z" name: isvc-xgboost-graph-predictor.18a960e0e11f1629 namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: d2192c74-05a8-4f37-a3ff-dcd7ae732368 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: 8963827d-bb6d-4a81-9aaf-5ccb73206ac4 kind: Event lastTimestamp: "2026-04-24T19:15:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:37Z" name: isvc-xgboost-graph-predictor.18a960eb5d5dd46b namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: f3acc2a6-bfb6-4419-9754-5fc3ad58d020 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: 8963827d-bb6d-4a81-9aaf-5ccb73206ac4 kind: Event lastTimestamp: "2026-04-24T19:15:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:37Z" name: isvc-xgboost-graph-predictor.18a960eb5da50200 namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 90d2954a-d373-418f-ace5-3e49ad01ab6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13142" uid: d4a8a825-8f04-4b78-876b-f9fa40f021ff kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:23Z" name: isvc-xgboost-graph.18a960dd62024fb6 namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: ffd9973f-ee86-4095-9b0e-183ebfafeb4e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: d4a8a825-8f04-4b78-876b-f9fa40f021ff kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:23Z" name: isvc-xgboost-graph.18a960f2c9675a87 namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 6dbce7f7-964e-4bdf-9857-6a0fe89bd211 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:15:51.694862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5cd78cfd67-m6wn9 to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:15:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:15:51Z" name: model-chainer-5cd78cfd67-m6wn9.18a960f944fc1e18 namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: fe7e1cb0-3b16-4f58-aba5-17d7c869353b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:51Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:15:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:51Z" name: model-chainer-5cd78cfd67-m6wn9.18a960f94dce76bf namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 0a7ce846-818e-48f0-83ac-35599329b177 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:52Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:15:52Z" name: model-chainer-5cd78cfd67-m6wn9.18a960f981822f20 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: afb05645-087f-4225-a4c4-ffe2ce6c4fcf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:52Z" name: model-chainer-5cd78cfd67-m6wn9.18a960f9832dce0f namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 01e18cd8-4e8d-45ff-8fa7-563330aa51b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-24T19:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:52Z" name: model-chainer-5cd78cfd67-m6wn9.18a960f986499d93 namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 67cc54cb-b904-47e8-a368-2ac9f7492e1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-24T19:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:52Z" name: model-chainer-5cd78cfd67-m6wn9.18a960f986b0275a namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 08b8af9c-bf8b-414f-82e8-8d31c01f2063 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:01Z" name: model-chainer-5cd78cfd67-m6wn9.18a960fb9e9829d7 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 5b8782d7-28c6-45b5-8a34-59a97e2ee196 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5cd78cfd67-m6wn9 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 489ddaf5-7441-48ad-aee3-efe71a0a941e kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: model-chainer-5cd78cfd67-m6wn9.18a960fc25b614ed namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: fd5f21df-0bf7-44d8-88f4-136e1cd28611 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5cd78cfd67 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 925d99c7-686d-43b4-9b2a-922b5d3fb874 kind: Event lastTimestamp: "2026-04-24T19:15:51Z" message: 'Created pod: model-chainer-5cd78cfd67-m6wn9' metadata: creationTimestamp: "2026-04-24T19:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:51Z" name: model-chainer-5cd78cfd67.18a960f94495fc64 namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: b938fdac-3adf-42f6-aea1-412d189f5253 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 1d7df7a8-28be-4585-aad7-2f0045924d15 kind: Event lastTimestamp: "2026-04-24T19:15:51Z" message: Scaled up replica set model-chainer-5cd78cfd67 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:51Z" name: model-chainer.18a960f943e791cf namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 82adb649-f582-4903-a2aa-af8ee9af3871 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: d475085c-56a4-411c-a35e-81a157960f84 kind: Event lastTimestamp: "2026-04-24T19:15:51Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-24T19:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:51Z" name: model-chainer.18a960f948ddbdcd namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: daba98d0-11ba-4429-8d4b-e241bdbb43e0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:26:16.821772Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-443ff-6999dd8899-xg485 to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:26:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:26:16Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618ad173caab namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 7cb883fe-4d45-4cc4-b053-683108b1caa5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:26:17Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618aeafd14ad namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 8b78c0b7-38ca-4742-bef4-ea33bb49b737 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-443ff} kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:17Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618aec3941a3 namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: b50d3b04-e095-436e-bc7e-0c67830c7b65 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-443ff} kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: 'Created container: sequence-graph-443ff' metadata: creationTimestamp: "2026-04-24T19:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:17Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618aef4c6e60 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 2a20198e-00c2-40b6-b962-404b5a5b49d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-443ff} kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: Started container sequence-graph-443ff metadata: creationTimestamp: "2026-04-24T19:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:17Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618af0072a3e namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 2e0d496a-fbc9-49f4-9c51-7db1a1a78e77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-443ff} kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: "2026-04-24T19:26:26Z" message: Stopping container sequence-graph-443ff metadata: creationTimestamp: "2026-04-24T19:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:26Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618d29f5481c namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 89ee2c8e-d291-4bae-a729-608e8de72c08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-443ff} kind: Pod name: sequence-graph-443ff-6999dd8899-xg485 namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 5fd9b4a1-ee35-4a19-b6d7-b4ffbcd5cd8b kind: Event lastTimestamp: "2026-04-24T19:26:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:53Z" name: sequence-graph-443ff-6999dd8899-xg485.18a9618da0bca0b8 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 1dcb0f36-2f07-4c32-906c-0bbc9b3380c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-443ff-6999dd8899 namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 19312a4b-792c-4778-8275-8943c735b430 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: 'Created pod: sequence-graph-443ff-6999dd8899-xg485' metadata: creationTimestamp: "2026-04-24T19:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:16Z" name: sequence-graph-443ff-6999dd8899.18a9618ad10e7eb8 namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: ed75dc79-3f0e-43b7-a915-e884ca551e9d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-443ff namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: f0dffd50-ee59-4ab8-91e3-0931ed5684c4 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: Scaled up replica set sequence-graph-443ff-6999dd8899 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:16Z" name: sequence-graph-443ff.18a9618ad0779523 namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 355dd697-38cc-41e5-879f-e8bc6502cb66 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-443ff namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: b4280849-ab17-4791-850b-70450efa9702 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: InferenceGraph [sequence-graph-443ff] is Ready metadata: creationTimestamp: "2026-04-24T19:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:16Z" name: sequence-graph-443ff.18a9618ad43982df namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 0e1669a2-8797-4866-8733-a297040101e5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:17:01.981736Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-99560-7846699b87-ljhdf to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:17:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:17:01Z" name: sequence-graph-99560-7846699b87-ljhdf.18a96109a268c0f2 namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: 7d45bc6f-d8cf-4728-8c56-3b54408d0fa3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:17:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-99560-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:17:02Z" name: sequence-graph-99560-7846699b87-ljhdf.18a96109b16e9f29 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 5bcb4b3e-2d26-44b8-b100-625498a513b8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:17:02Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:17:02Z" name: sequence-graph-99560-7846699b87-ljhdf.18a96109dec93931 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: ee386dec-4b1b-447e-bd36-dd2dc37a2aca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99560} kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:17:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:17:03Z" name: sequence-graph-99560-7846699b87-ljhdf.18a96109e0368ddd namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: a2d64eb1-7136-4593-92b6-198bdcb8c13b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99560} kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:17:03Z" message: 'Created container: sequence-graph-99560' metadata: creationTimestamp: "2026-04-24T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:17:03Z" name: sequence-graph-99560-7846699b87-ljhdf.18a96109e2ee318c namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: d4e565f5-4fe1-4736-bda1-3afe4aee7583 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99560} kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:17:03Z" message: Started container sequence-graph-99560 metadata: creationTimestamp: "2026-04-24T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:17:03Z" name: sequence-graph-99560-7846699b87-ljhdf.18a96109e35a0ed8 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 24f9e986-8596-48a7-8b75-4293186f11ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99560} kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Stopping container sequence-graph-99560 metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:16Z" name: sequence-graph-99560-7846699b87-ljhdf.18a9617ccfdc97bc namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 5cce888f-a24f-4fc8-af9b-32c927441ade reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:25:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-99560} kind: Pod name: sequence-graph-99560-7846699b87-ljhdf namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 298f3ec6-a358-4e20-a59b-26c36c657012 kind: Event lastTimestamp: "2026-04-24T19:25:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:44Z" name: sequence-graph-99560-7846699b87-ljhdf.18a9617d6b392660 namespace: kserve-ci-e2e-test resourceVersion: "19416" uid: 9f358c52-069f-41a9-ba65-c5e7a6be0cf4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-99560-7846699b87 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 5d13378b-215f-4ecf-a8ab-b8cc36da6493 kind: Event lastTimestamp: "2026-04-24T19:17:01Z" message: 'Created pod: sequence-graph-99560-7846699b87-ljhdf' metadata: creationTimestamp: "2026-04-24T19:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:01Z" name: sequence-graph-99560-7846699b87.18a96109a2070cc8 namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 9edff73b-2344-4400-8e7f-2c7c549760a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-99560 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: a64672d3-b4cf-402a-a60e-69b6d4126ccf kind: Event lastTimestamp: "2026-04-24T19:17:01Z" message: Scaled up replica set sequence-graph-99560-7846699b87 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:01Z" name: sequence-graph-99560.18a96109a121827f namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4e4629d2-8c6e-4e5a-b810-6afe13be5631 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-99560 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 841cd08d-d570-4710-a7bb-afab06a70610 kind: Event lastTimestamp: "2026-04-24T19:17:02Z" message: InferenceGraph [sequence-graph-99560] is Ready metadata: creationTimestamp: "2026-04-24T19:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:17:02Z" name: sequence-graph-99560.18a96109a6d440aa namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: abf5474e-61dc-499e-aa39-f0304890f327 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:17:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-99560 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: b0082112-f434-4228-8406-21af290aaf81 kind: Event lastTimestamp: "2026-04-24T19:17:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:31Z" name: sequence-graph-99560.18a9610d1fab02c0 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 194d2045-1bd3-48ad-9f56-7871fe052031 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:17:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-99560 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: b0082112-f434-4228-8406-21af290aaf81 kind: Event lastTimestamp: "2026-04-24T19:17:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:31Z" name: sequence-graph-99560.18a9610d1fe9cb99 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 366a1f1f-68e1-46a9-b524-9718b3224617 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:27:27.124867Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-cb3b2-5bb4f7479d-567lv to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9619b2fd7db3f namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: e5ba0f63-5463-4f7c-8be9-1ec3f1bc815a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9619b48a1cf45 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 75f6c2af-9f2b-4f3f-97a2-3a8dc3089f8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cb3b2} kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9619b49cf5a88 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 9d0a609d-5eb3-4e36-8173-cb531b0e1490 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cb3b2} kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: 'Created container: sequence-graph-cb3b2' metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9619b4cb9cc24 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 488d95d6-b655-4873-9570-82ba2c652d7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cb3b2} kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: Started container sequence-graph-cb3b2 metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9619b4d29b003 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 18c0f859-6846-47c4-8774-b972931f4005 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cb3b2} kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Stopping container sequence-graph-cb3b2 metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:41Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9620e5ad2ef3a namespace: kserve-ci-e2e-test resourceVersion: "24272" uid: bac59c34-7661-4556-b181-366b4a773373 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:35:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cb3b2} kind: Pod name: sequence-graph-cb3b2-5bb4f7479d-567lv namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 7aad4d3e-eb9d-4c81-a294-e14d72ffd25b kind: Event lastTimestamp: "2026-04-24T19:36:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:35:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:09Z" name: sequence-graph-cb3b2-5bb4f7479d-567lv.18a9620ee65ff00c namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: b64dc27a-dc69-4a1b-9d08-2a0a5ddef4d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-cb3b2-5bb4f7479d namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: d2f3c6e4-a654-4ff1-9970-f0b03493d52b kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: 'Created pod: sequence-graph-cb3b2-5bb4f7479d-567lv' metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2-5bb4f7479d.18a9619b2f4a5b9c namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 536be223-b353-4642-875a-220dcaf97efd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: d244e90f-9018-408e-9a73-29ed87a09d1c kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: Scaled up replica set sequence-graph-cb3b2-5bb4f7479d from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2.18a9619b2ea85d94 namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: f3de384c-1bca-47be-9bad-30060a13c29c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9fd702f9-a089-4303-b39b-45f38e5fb31d kind: Event lastTimestamp: "2026-04-24T19:27:27Z" message: InferenceGraph [sequence-graph-cb3b2] is Ready metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:27Z" name: sequence-graph-cb3b2.18a9619b32541a92 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: ca0cefee-d449-4600-8dd9-345b4d4441a4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 1aa9332d-83be-4c5e-8488-f798d50c50c1 kind: Event lastTimestamp: "2026-04-24T19:28:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:28:12Z" name: sequence-graph-cb3b2.18a9619ead5f8222 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 0157015c-993d-4cc8-9086-a93be5623bc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 1aa9332d-83be-4c5e-8488-f798d50c50c1 kind: Event lastTimestamp: "2026-04-24T19:28:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:28:12Z" name: sequence-graph-cb3b2.18a9619eadcc3c5d namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: a6453530-eecd-41bf-a789-8c7d90d5cda5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:37:22.537408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-11f3c-5fc8d754fc-jxw7p to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:37:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:37:22Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a96225d1323b57 namespace: kserve-ci-e2e-test resourceVersion: "25497" uid: b1d20776-b8f9-4128-afb1-b3f12f7253d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:22Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: "2026-04-24T19:37:22Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:37:22Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a96225ea0785e6 namespace: kserve-ci-e2e-test resourceVersion: "25513" uid: 0ca0e849-fb85-491b-9349-84f105fcc01b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11f3c} kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: "2026-04-24T19:37:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:37:22Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a96225eb40afa4 namespace: kserve-ci-e2e-test resourceVersion: "25515" uid: c428505e-1e5c-4708-b4ec-7cc4432c87b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11f3c} kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: "2026-04-24T19:37:23Z" message: 'Created container: splitter-graph-11f3c' metadata: creationTimestamp: "2026-04-24T19:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:37:23Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a96225ee48f647 namespace: kserve-ci-e2e-test resourceVersion: "25517" uid: d1bacc43-6544-4c72-a9bb-45b4cdacdf35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11f3c} kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: "2026-04-24T19:37:23Z" message: Started container splitter-graph-11f3c metadata: creationTimestamp: "2026-04-24T19:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:37:23Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a96225eebab3ac namespace: kserve-ci-e2e-test resourceVersion: "25518" uid: dd329c61-391e-44e8-a25a-129d0d7c9f0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11f3c} kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: "2026-04-24T19:45:37Z" message: Stopping container splitter-graph-11f3c metadata: creationTimestamp: "2026-04-24T19:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:37Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a96298fab2e0fe namespace: kserve-ci-e2e-test resourceVersion: "29253" uid: 4a251ae0-7573-4362-83ca-fc1365afbc21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-11f3c} kind: Pod name: splitter-graph-11f3c-5fc8d754fc-jxw7p namespace: kserve-ci-e2e-test resourceVersion: "25494" uid: 7bf929f1-6853-43f5-8512-c70688f4852c kind: Event lastTimestamp: "2026-04-24T19:46:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:46:04Z" name: splitter-graph-11f3c-5fc8d754fc-jxw7p.18a962999b55b9e8 namespace: kserve-ci-e2e-test resourceVersion: "29495" uid: 7d48e58a-179f-4496-b636-e396bcad9ea1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-11f3c-5fc8d754fc namespace: kserve-ci-e2e-test resourceVersion: "25489" uid: 82701005-07dc-41b2-91bc-e4596ca7a125 kind: Event lastTimestamp: "2026-04-24T19:37:22Z" message: 'Created pod: splitter-graph-11f3c-5fc8d754fc-jxw7p' metadata: creationTimestamp: "2026-04-24T19:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:22Z" name: splitter-graph-11f3c-5fc8d754fc.18a96225d0c2fb82 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 7f9e6808-ad34-4ffd-acd2-22f86b9ec2d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-11f3c namespace: kserve-ci-e2e-test resourceVersion: "25487" uid: b956ada2-458e-4c2b-bc6d-14608f2694e5 kind: Event lastTimestamp: "2026-04-24T19:37:22Z" message: Scaled up replica set splitter-graph-11f3c-5fc8d754fc from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:22Z" name: splitter-graph-11f3c.18a96225cfec596b namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: c3e034e0-1092-4b90-abef-b34d1e10cbb4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-11f3c namespace: kserve-ci-e2e-test resourceVersion: "25505" uid: b20e218a-c531-4817-8cd1-5c849922347c kind: Event lastTimestamp: "2026-04-24T19:37:22Z" message: InferenceGraph [splitter-graph-11f3c] is Ready metadata: creationTimestamp: "2026-04-24T19:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:37:22Z" name: splitter-graph-11f3c.18a96225d43f570e namespace: kserve-ci-e2e-test resourceVersion: "25507" uid: 1e4cf23b-7e16-4257-9fbc-c088943729ff reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:37:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-11f3c namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: b33ad876-ca25-4945-8b75-3198ca306ae8 kind: Event lastTimestamp: "2026-04-24T19:37:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:52Z" name: splitter-graph-11f3c.18a962294e38c494 namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: 8d9cfdce-3256-4afd-a8c6-b2165bcfbd65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:37:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-11f3c namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: b33ad876-ca25-4945-8b75-3198ca306ae8 kind: Event lastTimestamp: "2026-04-24T19:37:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:52Z" name: splitter-graph-11f3c.18a962294ea741a5 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 17d8083e-d547-40e7-bee9-37938cc3843f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:36:12.261150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24680" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-8e956-6bb6cf5f5d-dbpqh to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:36:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:36:12Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a962157467ad81 namespace: kserve-ci-e2e-test resourceVersion: "24686" uid: 24f56fa9-a0ab-4b6b-8910-f9e39262caf4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:36:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-8e956-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:12Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a962157cc729ed namespace: kserve-ci-e2e-test resourceVersion: "24705" uid: c73d1568-e5dc-4d3f-9d0e-22f9fca6d9c6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:14Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24691" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:14Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:36:14Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a96215e6a7b864 namespace: kserve-ci-e2e-test resourceVersion: "24717" uid: ab8944cc-5b0b-4e7e-98ad-cc22019ad800 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8e956} kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:14Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a96215e7d09739 namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: b724f967-ab26-4e17-821f-87852a70d019 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8e956} kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:14Z" message: 'Created container: splitter-graph-8e956' metadata: creationTimestamp: "2026-04-24T19:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:14Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a96215eac21c47 namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: 83762192-f742-4de0-9525-9808298f5248 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8e956} kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:14Z" message: Started container splitter-graph-8e956 metadata: creationTimestamp: "2026-04-24T19:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:14Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a96215eb339722 namespace: kserve-ci-e2e-test resourceVersion: "24721" uid: bc2f5752-7e1d-4b64-812b-0196641c2164 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8e956} kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Stopping container splitter-graph-8e956 metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a96217ccc128b6 namespace: kserve-ci-e2e-test resourceVersion: "24805" uid: 03a47273-72ff-40b1-a4f9-64e42ae2c995 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8e956} kind: Pod name: splitter-graph-8e956-6bb6cf5f5d-dbpqh namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9d781b40-a10f-49f3-b1d7-1bcb7915eab4 kind: Event lastTimestamp: "2026-04-24T19:36:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:50Z" name: splitter-graph-8e956-6bb6cf5f5d-dbpqh.18a9621891d863ec namespace: kserve-ci-e2e-test resourceVersion: "25173" uid: db6a5ebd-f82e-4206-933d-855d5dccefba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-8e956-6bb6cf5f5d namespace: kserve-ci-e2e-test resourceVersion: "24677" uid: 3c2cc5d6-3c9a-46e4-847e-ba7c62b60f92 kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: 'Created pod: splitter-graph-8e956-6bb6cf5f5d-dbpqh' metadata: creationTimestamp: "2026-04-24T19:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:12Z" name: splitter-graph-8e956-6bb6cf5f5d.18a9621573dbe931 namespace: kserve-ci-e2e-test resourceVersion: "24682" uid: 49036362-7a89-4ba1-9bc7-81502360e213 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-8e956 namespace: kserve-ci-e2e-test resourceVersion: "24675" uid: 59465e72-0512-4f99-9a45-53668cc0d6b8 kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: Scaled up replica set splitter-graph-8e956-6bb6cf5f5d from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:12Z" name: splitter-graph-8e956.18a9621572d4b052 namespace: kserve-ci-e2e-test resourceVersion: "24678" uid: 4b7bb65e-30a7-40c7-9f24-210b4f4ad69c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-8e956 namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: 72008970-d682-4e24-9366-6afa030f5abd kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: InferenceGraph [splitter-graph-8e956] is Ready metadata: creationTimestamp: "2026-04-24T19:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:12Z" name: splitter-graph-8e956.18a9621578280135 namespace: kserve-ci-e2e-test resourceVersion: "24695" uid: 9735bfde-e474-4dcb-a689-91d3c86d5812 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:36:22.455257Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24842" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217d40561f5 namespace: kserve-ci-e2e-test resourceVersion: "24851" uid: cfc4b1ca-e73c-49e0-bfb7-2bc104783a1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24865" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217ee0c3fa8 namespace: kserve-ci-e2e-test resourceVersion: "24900" uid: d13185ee-a1d8-41e1-a1fc-d3fca6330200 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217ef559992 namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: a2e8e362-488b-426e-a64b-ff76efbb153d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217f1ee139e namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: 9f81b1a8-5155-4619-b86a-e0291198d1cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217f265f3b7 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: ce5a5b64-29cb-469a-abf8-a87339e19393 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217f2735821 namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: 6a04aa6f-9d47-4332-b001-a3d2c8b54f82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217f5319c12 namespace: kserve-ci-e2e-test resourceVersion: "24908" uid: f223e84f-efb4-43af-b77d-17c57832b6f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:36:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:23Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96217f5a87f8e namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: f0b1ad54-7edd-4285-8713-efbe65425afa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:45:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:39Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a962182155540e namespace: kserve-ci-e2e-test resourceVersion: "29300" uid: 61c4c2da-8850-4bc4-b9a9-833fa3af4fed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:45:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:37Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96298ff1ba523 namespace: kserve-ci-e2e-test resourceVersion: "29266" uid: 80038659-7256-4c78-8ee4-5d54f23eda70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:45:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:37Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a96298ff1c3e8e namespace: kserve-ci-e2e-test resourceVersion: "29268" uid: 3264a20e-4087-4db2-9fa0-d308599022d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: dc6093af-e5c9-4707-9146-0e5c8e6b48b3 kind: Event lastTimestamp: "2026-04-24T19:45:39Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:45:39Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm.18a9629995a04dd7 namespace: kserve-ci-e2e-test resourceVersion: "29299" uid: 93fd3669-58b9-419c-8522-f95c3eb93c6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-11f3c-predictor-b9f5b46d7 namespace: kserve-ci-e2e-test resourceVersion: "24831" uid: 63c56a27-b05e-4a63-a11a-928d98cbb9e4 kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'Created pod: success-200-isvc-11f3c-predictor-b9f5b46d7-btmfm' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor-b9f5b46d7.18a96217d37b0614 namespace: kserve-ci-e2e-test resourceVersion: "24844" uid: 38c5d5d0-ee15-4551-91df-88415a1e6c9b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24828" uid: 4dc7c326-49d7-448f-97f7-23e1d38fa96a kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Scaled up replica set success-200-isvc-11f3c-predictor-b9f5b46d7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c-predictor.18a96217d1e3d8a8 namespace: kserve-ci-e2e-test resourceVersion: "24834" uid: 5b02bd61-a393-4971-ade4-ccedad091daf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:36:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24830" uid: f0abc991-7411-4ce0-b7f0-f0c4bd7e1f98 kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:52Z" name: success-200-isvc-11f3c-predictor.18a9621b50c85909 namespace: kserve-ci-e2e-test resourceVersion: "25216" uid: 65773746-eac2-45e2-8157-c7f6d2db2086 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:36:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24830" uid: f0abc991-7411-4ce0-b7f0-f0c4bd7e1f98 kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:52Z" name: success-200-isvc-11f3c-predictor.18a9621b51018eff namespace: kserve-ci-e2e-test resourceVersion: "25217" uid: 6ad19389-0079-4df0-baf9-9b0496a0c3f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:37:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: f0abc991-7411-4ce0-b7f0-f0c4bd7e1f98 kind: Event lastTimestamp: "2026-04-24T19:37:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:37Z" name: success-200-isvc-11f3c-predictor.18a962224e575ba0 namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: b67c8c7f-804a-4db5-b7a4-1aed7a638201 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:37:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-11f3c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: f0abc991-7411-4ce0-b7f0-f0c4bd7e1f98 kind: Event lastTimestamp: "2026-04-24T19:37:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:37:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:37Z" name: success-200-isvc-11f3c-predictor.18a962224eb5d45f namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 117177d5-2402-4740-89f9-5cc7743c146a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "24808" uid: 10072cc1-6d46-4609-986a-fce859e4a48b kind: Event lastTimestamp: "2026-04-24T19:37:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:37:09Z" name: success-200-isvc-11f3c.18a96217d1bc2187 namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: 3e865dfd-2223-493f-839e-a700e58f5d9e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "24808" uid: 10072cc1-6d46-4609-986a-fce859e4a48b kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'Failed to update status for InferenceService "success-200-isvc-11f3c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-11f3c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c.18a96217d223b2e3 namespace: kserve-ci-e2e-test resourceVersion: "24837" uid: aff9f539-b629-4ec6-b6da-c852beaad5ed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "24808" uid: 10072cc1-6d46-4609-986a-fce859e4a48b kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-11f3c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-11f3c.18a96217d2240728 namespace: kserve-ci-e2e-test resourceVersion: "24839" uid: 6773b732-0699-4c9f-892e-5d1bfabcfb0a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:37:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-11f3c namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 10072cc1-6d46-4609-986a-fce859e4a48b kind: Event lastTimestamp: "2026-04-24T19:37:09Z" message: InferenceService [success-200-isvc-11f3c] is Ready metadata: creationTimestamp: "2026-04-24T19:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:37:09Z" name: success-200-isvc-11f3c.18a96222db967ec6 namespace: kserve-ci-e2e-test resourceVersion: "25375" uid: a9e7cade-8a83-4739-a5ad-0ebbdbb80abe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:16.810806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-443ff-predictor-f8c98b6df-98lqm to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cd88547c0 namespace: kserve-ci-e2e-test resourceVersion: "19073" uid: 666360f1-a56b-4c0b-8cb2-fbedc3133d2d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19080" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cf203f7be namespace: kserve-ci-e2e-test resourceVersion: "19112" uid: 34da7b41-3ead-4a35-aab2-447b1d3fad75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cf35f61c2 namespace: kserve-ci-e2e-test resourceVersion: "19114" uid: e8b5608a-3947-4e29-825f-61d44fe6fb9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cf88cf2fb namespace: kserve-ci-e2e-test resourceVersion: "19119" uid: c2f3941e-4071-4476-94e1-774d2595aeda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cf914b029 namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: 86b0a84f-fdca-4614-9bbe-93397d54af4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cf920e16d namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: 94d0b6b2-5e73-45b4-b900-72bd5048ff7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cfb63fdf2 namespace: kserve-ci-e2e-test resourceVersion: "19124" uid: a19cab98-d9ea-46b5-8b5a-49322d201066 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:17Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617cfbd97b65 namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: f1639365-9df6-4ba6-8f42-c194f52138b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:25:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:54Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9617d4df0df62 namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 27eeda94-e218-4106-a4ab-30242f1371a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9618d30b30b61 namespace: kserve-ci-e2e-test resourceVersion: "20029" uid: d95e3192-6ad5-46e1-8338-6d605d72419c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9618d30b3eba8 namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 24a2dc6d-bb15-4408-8de1-287e48b86a8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 4cde8196-8e50-4150-a320-a610187ae842 kind: Event lastTimestamp: "2026-04-24T19:26:29Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:26:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:29Z" name: success-200-isvc-443ff-predictor-f8c98b6df-98lqm.18a9618dd5fd5b9b namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 20a61fd1-7950-4ca8-b6cb-d9a83ea97501 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-443ff-predictor-f8c98b6df namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: e1758fd8-46d0-476c-89c2-293c2df35d5e kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: 'Created pod: success-200-isvc-443ff-predictor-f8c98b6df-98lqm' metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-443ff-predictor-f8c98b6df.18a9617cd8217863 namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: c2fe7b1a-5466-4649-803f-ef8d2e1e6c52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19050" uid: 0314fe1e-e54a-41c7-a788-029716453048 kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Scaled up replica set success-200-isvc-443ff-predictor-f8c98b6df from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-443ff-predictor.18a9617cd605877c namespace: kserve-ci-e2e-test resourceVersion: "19060" uid: 8fac5915-55aa-45d9-a331-35178ff19ee1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: 05be507e-aa4e-4f2d-8259-827e4ee14ca2 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:16Z" name: success-200-isvc-443ff-predictor.18a96180546a21e7 namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: f30ebdc7-abb5-450f-b14e-cac4ef311b19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: 05be507e-aa4e-4f2d-8259-827e4ee14ca2 kind: Event lastTimestamp: "2026-04-24T19:26:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:16Z" name: success-200-isvc-443ff-predictor.18a9618054a946e6 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: b09acbf8-3d3f-4f69-8bfc-9bd2e0c82ece reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: 05be507e-aa4e-4f2d-8259-827e4ee14ca2 kind: Event lastTimestamp: "2026-04-24T19:25:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:46Z" name: success-200-isvc-443ff-predictor.18a96183d3416760 namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: e19ce7c5-2824-42c2-b2de-2a6fd3a7bda9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443ff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: 05be507e-aa4e-4f2d-8259-827e4ee14ca2 kind: Event lastTimestamp: "2026-04-24T19:25:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:46Z" name: success-200-isvc-443ff-predictor.18a96183d3a50d71 namespace: kserve-ci-e2e-test resourceVersion: "19439" uid: a612d9ff-d156-4a3c-ab2d-37f70b7362ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-443ff namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: 9b912add-685d-4d39-9272-b1b8c98a2d2b kind: Event lastTimestamp: "2026-04-24T19:26:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:04Z" name: success-200-isvc-443ff.18a9617cd5c8d2ae namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: d7da0de2-0628-413b-a628-0ec6d9015e54 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-443ff namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: 9b912add-685d-4d39-9272-b1b8c98a2d2b kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-443ff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-443ff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-443ff.18a9617cd61c924a namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: 1dddee6a-fe93-4fbb-a562-f5616d761032 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-443ff namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: 9b912add-685d-4d39-9272-b1b8c98a2d2b kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-443ff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-443ff.18a9617cd61cf956 namespace: kserve-ci-e2e-test resourceVersion: "19064" uid: 61035fb7-f15f-47fd-81ff-2c884fba64d6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-443ff namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 9b912add-685d-4d39-9272-b1b8c98a2d2b kind: Event lastTimestamp: "2026-04-24T19:26:04Z" message: InferenceService [success-200-isvc-443ff] is Ready metadata: creationTimestamp: "2026-04-24T19:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:04Z" name: success-200-isvc-443ff.18a961880a119a61 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: d64a9a23-8883-4e90-acc3-1a0673047707 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:13:51.871172Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13138" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:13:51Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960dd5eefc321 namespace: kserve-ci-e2e-test resourceVersion: "13143" uid: ee9dcaa7-4363-42a6-852b-5e4d09938001 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-48e5e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:52Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960dd6b60f368 namespace: kserve-ci-e2e-test resourceVersion: "13190" uid: c4b4e270-bf19-44cf-a387-e6d027079344 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13147" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:13:52Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960dd9bc1d6d7 namespace: kserve-ci-e2e-test resourceVersion: "13245" uid: 7f67b513-440a-4fc0-9f85-271ada5ef77e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:13:52Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" metadata: creationTimestamp: "2026-04-24T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:13:52Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960dd9d2ece62 namespace: kserve-ci-e2e-test resourceVersion: "13247" uid: bb3354ff-2c8b-404a-ab4d-386783ba9873 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" in 15.746s (15.746s including waiting). Image size: 1337635937 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e147c49f3f namespace: kserve-ci-e2e-test resourceVersion: "13431" uid: 0f661671-62da-46a0-ab22-98da0f58b084 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e14ba024e6 namespace: kserve-ci-e2e-test resourceVersion: "13434" uid: 9a6987ba-3c32-4fc0-8959-8a96a5ece7ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e14c19b6ab namespace: kserve-ci-e2e-test resourceVersion: "13437" uid: dca0d2fc-361c-49d0-a7b6-80311941d965 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T19:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:08Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e14c254c6d namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: c18cb095-9a89-44f6-b056-323f1b0316fa reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:11Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.571s (2.571s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:11Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e1e56c5e56 namespace: kserve-ci-e2e-test resourceVersion: "13457" uid: b13d9373-642f-442f-9915-daf398241647 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:11Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e1e91c51d4 namespace: kserve-ci-e2e-test resourceVersion: "13459" uid: 6406170f-b506-49de-8229-4ff60f3f7f79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:11Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e1e99617d0 namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 28b38401-74b2-481f-87a3-a15d62046ff8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:14:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:48Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960e241839771 namespace: kserve-ci-e2e-test resourceVersion: "13802" uid: ba2dad66-cbd0-4c21-bc00-160fb8eab149 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960f25803f51c namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: 964243bf-c03c-46ce-8d0b-adbf05073e84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960f258048f75 namespace: kserve-ci-e2e-test resourceVersion: "14165" uid: 7121003b-93e8-44fa-99db-d5854b4feaf6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx namespace: kserve-ci-e2e-test resourceVersion: "13140" uid: 03f34821-7441-4348-8dcf-d4abd092e54b kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx.18a960f2c9a2bf59 namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: 0afbc47b-11da-4938-bdc3-a850ab78904c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-48e5e-predictor-85bc6cd4f namespace: kserve-ci-e2e-test resourceVersion: "13131" uid: 109d8f28-34c5-4889-bead-a23fd2432f6d kind: Event lastTimestamp: "2026-04-24T19:13:51Z" message: 'Created pod: success-200-isvc-48e5e-predictor-85bc6cd4f-z28gx' metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:51Z" name: success-200-isvc-48e5e-predictor-85bc6cd4f.18a960dd5e80a0cc namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: a64149b2-1b6f-4310-a8e6-bb9bbffa8349 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13129" uid: 1d66daca-205d-424a-b637-1726dd7f0b13 kind: Event lastTimestamp: "2026-04-24T19:13:51Z" message: Scaled up replica set success-200-isvc-48e5e-predictor-85bc6cd4f from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:13:51Z" name: success-200-isvc-48e5e-predictor.18a960dd5d02f9d9 namespace: kserve-ci-e2e-test resourceVersion: "13134" uid: b332328c-11d4-40ea-80c2-478ecb3db012 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13130" uid: 96798e65-3e73-45e1-9df8-367d26810352 kind: Event lastTimestamp: "2026-04-24T19:14:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:36Z" name: success-200-isvc-48e5e-predictor.18a960e0db7db491 namespace: kserve-ci-e2e-test resourceVersion: "13703" uid: b5cf0a22-376f-4569-980a-e7ca64eecae4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13130" uid: 96798e65-3e73-45e1-9df8-367d26810352 kind: Event lastTimestamp: "2026-04-24T19:14:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:36Z" name: success-200-isvc-48e5e-predictor.18a960e0dbe9250a namespace: kserve-ci-e2e-test resourceVersion: "13704" uid: b919a0d8-b349-43e6-8eb9-c9f29c7d8c9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13411" uid: 96798e65-3e73-45e1-9df8-367d26810352 kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-48e5e-predictor.18a960eb57f5e526 namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 8bcc04bb-a019-4d69-bf2b-f012bd5615e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:14:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13411" uid: 96798e65-3e73-45e1-9df8-367d26810352 kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-48e5e-predictor.18a960eb585a941d namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: aa57a45f-2b6a-4083-bb6b-e13b422c95e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:13:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-48e5e namespace: kserve-ci-e2e-test resourceVersion: "13118" uid: 3da4cd33-1fda-4848-a90b-13e0ade96f78 kind: Event lastTimestamp: "2026-04-24T19:14:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:14:59Z" name: success-200-isvc-48e5e.18a960dd5cd276cc namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: 9e6c9c4e-926a-41c7-8789-3649aa66ebbf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-48e5e namespace: kserve-ci-e2e-test resourceVersion: "13902" uid: 3da4cd33-1fda-4848-a90b-13e0ade96f78 kind: Event lastTimestamp: "2026-04-24T19:14:58Z" message: InferenceService [success-200-isvc-48e5e] is Ready metadata: creationTimestamp: "2026-04-24T19:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:14:58Z" name: success-200-isvc-48e5e.18a960ecfe723c91 namespace: kserve-ci-e2e-test resourceVersion: "13905" uid: 63c66124-12aa-40e5-87f7-057cac104c35 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:35:12.211906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23955" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a962077930f03d namespace: kserve-ci-e2e-test resourceVersion: "23964" uid: b5764f85-60f3-41f7-b859-fda8bf65c28c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23976" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a9620792581488 namespace: kserve-ci-e2e-test resourceVersion: "23997" uid: 3b87ebe8-35e8-40b3-85a6-48cdfcbd255a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a96207939c74bf namespace: kserve-ci-e2e-test resourceVersion: "24000" uid: 7e03e33e-7cc6-42b8-8374-125be65e89fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a9620796a8a254 namespace: kserve-ci-e2e-test resourceVersion: "24002" uid: 9666395d-0bf1-4326-a7d6-81b494c4a055 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a96207971e879c namespace: kserve-ci-e2e-test resourceVersion: "24003" uid: e31a3c4e-f86c-4837-a16c-dde7e33f2ea6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a96207972b42f2 namespace: kserve-ci-e2e-test resourceVersion: "24004" uid: e1d7bea2-d12b-4391-ad8b-c29881404333 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a962079a793a51 namespace: kserve-ci-e2e-test resourceVersion: "24005" uid: 80a7e048-7486-4272-9890-83797b71c935 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a962079af45614 namespace: kserve-ci-e2e-test resourceVersion: "24006" uid: 26ba6840-cd9a-4a26-a77e-32a5ee12a16b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:35:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:49Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a96207c6a861ca namespace: kserve-ci-e2e-test resourceVersion: "24465" uid: e479e03f-a1a6-4a72-ac82-83c83b93a364 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a96217d1c51e67 namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: bf9ba125-a40d-4bee-9569-c28f35d17294 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:36:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:22Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a96217d1c5c346 namespace: kserve-ci-e2e-test resourceVersion: "24836" uid: 6cfc5611-a9c9-499f-a0d0-4652d2035870 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn namespace: kserve-ci-e2e-test resourceVersion: "23959" uid: 597f7cde-771c-4421-a1c8-c72bafa6833b kind: Event lastTimestamp: "2026-04-24T19:36:24Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:24Z" name: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn.18a962184ecc24f9 namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: 232d325e-c907-443b-b9a6-d8bc82786ee9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8e956-predictor-b8b4d68c7 namespace: kserve-ci-e2e-test resourceVersion: "23928" uid: a7eba5eb-fa59-419b-a0f3-384b304c01f1 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'Created pod: success-200-isvc-8e956-predictor-b8b4d68c7-s2pwn' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor-b8b4d68c7.18a9620778c6ece3 namespace: kserve-ci-e2e-test resourceVersion: "23958" uid: b0a73b1a-5056-47ea-88a9-55abdc858da9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "23925" uid: 242c2000-1c10-4081-814b-57232c0312cc kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Scaled up replica set success-200-isvc-8e956-predictor-b8b4d68c7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956-predictor.18a9620774fd2c4d namespace: kserve-ci-e2e-test resourceVersion: "23931" uid: 8a7bbb2a-9a9f-42ea-8327-600d8ac9ce8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:35:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "23927" uid: 492cb230-c205-4f45-a3e9-5dec21f365da kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-8e956-predictor.18a9620af38affc1 namespace: kserve-ci-e2e-test resourceVersion: "24362" uid: 559cbaeb-962b-4dbd-9df5-4b2c8908ac93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:35:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "23927" uid: 492cb230-c205-4f45-a3e9-5dec21f365da kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-8e956-predictor.18a9620af3e89094 namespace: kserve-ci-e2e-test resourceVersion: "24363" uid: 1d85606c-299e-4f32-937a-4143af5cd731 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:35:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "24144" uid: 492cb230-c205-4f45-a3e9-5dec21f365da kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:12Z" name: success-200-isvc-8e956-predictor.18a96211f1627386 namespace: kserve-ci-e2e-test resourceVersion: "24667" uid: 9df70338-c150-4586-83c7-24d71798d682 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:35:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e956-predictor namespace: kserve-ci-e2e-test resourceVersion: "24144" uid: 492cb230-c205-4f45-a3e9-5dec21f365da kind: Event lastTimestamp: "2026-04-24T19:36:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:12Z" name: success-200-isvc-8e956-predictor.18a96211f1c41a2f namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: 6389700c-a16c-449b-87cc-fde5f322c772 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "23905" uid: 5bd283d6-416a-4158-b705-e830c7295b96 kind: Event lastTimestamp: "2026-04-24T19:35:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:59Z" name: success-200-isvc-8e956.18a9620774d7f287 namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 291f45fe-325c-4c05-bf3c-9a676532f001 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "23905" uid: 5bd283d6-416a-4158-b705-e830c7295b96 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-8e956": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8e956": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956.18a96207751b2de3 namespace: kserve-ci-e2e-test resourceVersion: "23933" uid: 0cb2ffd5-955c-4323-bdf8-9e92c618e7e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "23905" uid: 5bd283d6-416a-4158-b705-e830c7295b96 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8e956": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-8e956.18a96207751b83b1 namespace: kserve-ci-e2e-test resourceVersion: "23935" uid: 44fcfe62-da8a-449c-bd46-e70c9145e993 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e956 namespace: kserve-ci-e2e-test resourceVersion: "24571" uid: 5bd283d6-416a-4158-b705-e830c7295b96 kind: Event lastTimestamp: "2026-04-24T19:35:59Z" message: InferenceService [success-200-isvc-8e956] is Ready metadata: creationTimestamp: "2026-04-24T19:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:59Z" name: success-200-isvc-8e956.18a96212838634f0 namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: 9a8670a0-ebf7-46e3-9e8d-23c41f8d7ba8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:16:01.929041Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-99560-predictor-6f5d77689c-g76x6 to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:16:01Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fba6fd5bce namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: aa625fd4-9b1c-4eaf-958a-a781a7666259 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-99560-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:02Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbb1dbfd85 namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: 65f5f7b4-d095-48fb-8ff9-ad73f39f388f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:02Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:16:02Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbe3f26f6b namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: c7f9e183-270f-4218-a976-aaf6da3fb692 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:02Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbe57dcb79 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 4583bee7-6a74-4bad-87b7-a5727bcbff5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbe93e702f namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: a8b5b886-3396-4c74-8f35-dc3ee0cc040c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbe9c62fa0 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: cde9448d-7c0d-4e8b-ae2f-76cf1106cbe0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbe9d2d894 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 81a6074b-c301-498a-b6fd-663946107838 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbecb21559 namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 100685cb-1a9b-4a4e-a00c-a3cfe417d7fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:16:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:03Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fbed43c9f4 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 4f85e138-b064-48be-b529-baa8034b04f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:25:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:20Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a960fc2d1a8c80 namespace: kserve-ci-e2e-test resourceVersion: "19164" uid: 603390bb-7d08-49e5-a344-755c58febc25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a9617cd66daa27 namespace: kserve-ci-e2e-test resourceVersion: "19063" uid: 9c64826b-1f39-418f-8bcd-88c581cd4007 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:25:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:16Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a9617cd66e511a namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: c1e8bfcd-4d27-4bb0-ab5e-397149c1ac3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99560-predictor-6f5d77689c-g76x6 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: b9c97214-f5d6-411a-9961-05c62b416798 kind: Event lastTimestamp: "2026-04-24T19:25:20Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:20Z" name: success-200-isvc-99560-predictor-6f5d77689c-g76x6.18a9617da16c8888 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: e4a9a7f1-0feb-4cdd-9e55-9368989c630c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-99560-predictor-6f5d77689c namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: ab58d5c2-8737-481a-9352-f4e03a5528b1 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: 'Created pod: success-200-isvc-99560-predictor-6f5d77689c-g76x6' metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:01Z" name: success-200-isvc-99560-predictor-6f5d77689c.18a960fba69611d8 namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: 8948385f-5d66-401d-ad2b-bd6076ed77fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-99560-predictor namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: 80863a6f-4dbb-495e-82cc-2b6ff555ddc8 kind: Event lastTimestamp: "2026-04-24T19:16:01Z" message: Scaled up replica set success-200-isvc-99560-predictor-6f5d77689c from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:01Z" name: success-200-isvc-99560-predictor.18a960fba4b37fe7 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 35f73bb5-0b0c-4391-9d80-e7b6be1422a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99560-predictor namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 14d18914-4d1b-44ea-97b0-a6ba538d9993 kind: Event lastTimestamp: "2026-04-24T19:17:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:16Z" name: success-200-isvc-99560-predictor.18a960ff2381a557 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 17cc2c2b-ae55-4a1b-8908-d4169ad9f479 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99560-predictor namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 14d18914-4d1b-44ea-97b0-a6ba538d9993 kind: Event lastTimestamp: "2026-04-24T19:17:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:16Z" name: success-200-isvc-99560-predictor.18a960ff23e76d6e namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: c94d925d-4d90-4930-be79-16810e7c9c20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99560 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: cc5853c3-c8ec-4899-aba8-a083bd5cc99c kind: Event lastTimestamp: "2026-04-24T19:16:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:50Z" name: success-200-isvc-99560.18a960fba4acb464 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 0f6c3ccb-9169-46c4-859d-d19e2b3e6602 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99560 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: cc5853c3-c8ec-4899-aba8-a083bd5cc99c kind: Event lastTimestamp: "2026-04-24T19:16:50Z" message: InferenceService [success-200-isvc-99560] is Ready metadata: creationTimestamp: "2026-04-24T19:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:50Z" name: success-200-isvc-99560.18a96106e76cbeb9 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 9dd5a10f-21ae-4b65-8b8b-d5f1e9078e64 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:35:41.891713Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24307" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a0351-predictor-575fcb594c-rxptj to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e623edc46 namespace: kserve-ci-e2e-test resourceVersion: "24316" uid: 0c667cfe-b656-4aa6-b345-1aaa88174699 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24327" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e7c2cb505 namespace: kserve-ci-e2e-test resourceVersion: "24369" uid: fbd1ec56-528f-4228-8008-c224baaacbea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e7da01ca0 namespace: kserve-ci-e2e-test resourceVersion: "24371" uid: 7a9d4be2-c227-4aa2-a014-ce953144565c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e8074e194 namespace: kserve-ci-e2e-test resourceVersion: "24374" uid: 91dde673-0537-433c-ac2d-fda85285c974 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e80f07a6c namespace: kserve-ci-e2e-test resourceVersion: "24375" uid: ed231593-a80c-41ad-a01a-3af999fc2d7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e80fdd6aa namespace: kserve-ci-e2e-test resourceVersion: "24376" uid: 5f679f5d-269c-4a06-a69a-a664e4263636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e841daa02 namespace: kserve-ci-e2e-test resourceVersion: "24378" uid: e3eda212-69e8-4b13-aa51-c5dd4750580d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:35:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:42Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e84f0d1ad namespace: kserve-ci-e2e-test resourceVersion: "24379" uid: 9dbfdbd0-585f-4172-897f-48dab9fcfc22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:36:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:28Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9620e8cce189e namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: 1f52ba70-7701-48b2-a9be-13afdfe1cb20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:53:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:11Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a96302c2c2e4d3 namespace: kserve-ci-e2e-test resourceVersion: "32432" uid: dbd0ebbc-5948-4af0-8c8e-88237259ced6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:53:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:11Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a96302c2c36812 namespace: kserve-ci-e2e-test resourceVersion: "32433" uid: 5adf38af-2915-4751-8054-6d837830bbfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a0351-predictor-575fcb594c-rxptj namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 39c2c5c2-4106-44f1-9132-cc5e3065e04a kind: Event lastTimestamp: "2026-04-24T19:53:13Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:13Z" name: success-200-isvc-a0351-predictor-575fcb594c-rxptj.18a9630341a422d2 namespace: kserve-ci-e2e-test resourceVersion: "32498" uid: 3a5d8b75-1eaf-431e-b476-ddb084b1242c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a0351-predictor-575fcb594c namespace: kserve-ci-e2e-test resourceVersion: "24297" uid: ad1714c8-5f48-42ea-a7d6-afecc3acb3e5 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: 'Created pod: success-200-isvc-a0351-predictor-575fcb594c-rxptj' metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-a0351-predictor-575fcb594c.18a9620e61c693fd namespace: kserve-ci-e2e-test resourceVersion: "24309" uid: d98baccf-56af-40e8-ad26-3885479976c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24294" uid: c94d62d4-0a6e-40e9-a60e-fd219fa585a4 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Scaled up replica set success-200-isvc-a0351-predictor-575fcb594c from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-a0351-predictor.18a9620e6031b01e namespace: kserve-ci-e2e-test resourceVersion: "24300" uid: e038aaa1-e294-40ae-90d2-b1c6fe64901e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24296" uid: 6b80bc4d-0161-4134-930b-822f309d7910 kind: Event lastTimestamp: "2026-04-24T19:36:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:11Z" name: success-200-isvc-a0351-predictor.18a96211dea4d06d namespace: kserve-ci-e2e-test resourceVersion: "24658" uid: 1164e52f-9b80-4bd0-84c1-eade372b6b13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24296" uid: 6b80bc4d-0161-4134-930b-822f309d7910 kind: Event lastTimestamp: "2026-04-24T19:36:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:11Z" name: success-200-isvc-a0351-predictor.18a96211dee59572 namespace: kserve-ci-e2e-test resourceVersion: "24659" uid: 1b2a8837-0f80-4adc-a5fc-7c352ba05edb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24525" uid: 6b80bc4d-0161-4134-930b-822f309d7910 kind: Event lastTimestamp: "2026-04-24T19:37:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:26Z" name: success-200-isvc-a0351-predictor.18a96218dca97eee namespace: kserve-ci-e2e-test resourceVersion: "25547" uid: 200fffb6-8f2a-42e4-82fe-866a3a33d993 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a0351-predictor namespace: kserve-ci-e2e-test resourceVersion: "24525" uid: 6b80bc4d-0161-4134-930b-822f309d7910 kind: Event lastTimestamp: "2026-04-24T19:37:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:26Z" name: success-200-isvc-a0351-predictor.18a96218dcf01738 namespace: kserve-ci-e2e-test resourceVersion: "25548" uid: 0d8d8a24-c58d-4578-8653-1a21a0da9e76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24274" uid: 8fe7eb02-d408-42d8-b672-f97a823de932 kind: Event lastTimestamp: "2026-04-24T19:36:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:38Z" name: success-200-isvc-a0351.18a9620e600523ff namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: e1817e4b-faed-479d-8a8e-84a91ed74bf1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24274" uid: 8fe7eb02-d408-42d8-b672-f97a823de932 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-a0351": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a0351": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-a0351.18a9620e606560a1 namespace: kserve-ci-e2e-test resourceVersion: "24302" uid: a6da168e-ec91-4ec1-90b6-e4e5c5437c1a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "24274" uid: 8fe7eb02-d408-42d8-b672-f97a823de932 kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a0351": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-a0351.18a9620e6065c4f0 namespace: kserve-ci-e2e-test resourceVersion: "24304" uid: ed65e546-8779-4dc5-b8ac-64c32c52fb2f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a0351 namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: 8fe7eb02-d408-42d8-b672-f97a823de932 kind: Event lastTimestamp: "2026-04-24T19:36:38Z" message: InferenceService [success-200-isvc-a0351] is Ready metadata: creationTimestamp: "2026-04-24T19:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:38Z" name: success-200-isvc-a0351.18a9621b9b374f65 namespace: kserve-ci-e2e-test resourceVersion: "25077" uid: 6404967a-104a-4f5d-afa6-b79d9324794b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:24:36.942438Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2e49-predictor-869fccf77f-86ffs to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173902e1b20 namespace: kserve-ci-e2e-test resourceVersion: "18633" uid: 494ea035-7352-40b7-b576-af8fa6200e4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18646" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173a9979766 namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: d4503a71-05b9-45ea-afa4-6502f7fb48cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173aafab888 namespace: kserve-ci-e2e-test resourceVersion: "18675" uid: 2f4cd98a-e503-4317-9620-4798afab860a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173af939ca3 namespace: kserve-ci-e2e-test resourceVersion: "18679" uid: 09bdf8fb-9264-4dc8-b13c-76692e245038 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173b04e2292 namespace: kserve-ci-e2e-test resourceVersion: "18680" uid: 2b16a174-cf8c-452e-a014-f3699679836f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173b0596368 namespace: kserve-ci-e2e-test resourceVersion: "18681" uid: c5a20e08-8834-49a6-96cc-1f4eb081ce0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173b2782a17 namespace: kserve-ci-e2e-test resourceVersion: "18685" uid: 246d3800-13e5-490c-a022-3c508e3e3e00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:24:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:37Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173b2f0aa68 namespace: kserve-ci-e2e-test resourceVersion: "18686" uid: b7e5094a-34ca-420b-b9b1-cb0b2566de92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:24:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:25:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:24:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:14Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96173f58e11d9 namespace: kserve-ci-e2e-test resourceVersion: "18983" uid: 0189ff3e-de6a-4ba9-835f-b218ab84eece reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96183edad2115 namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: c2092bb5-7292-4c5d-bcdd-93f0a03fc147 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a96183edade13a namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 95b88385-ef53-4ee5-924f-7eb72a0d9bcc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 6a243971-45ac-410a-9e5d-ef4761931a44 kind: Event lastTimestamp: "2026-04-24T19:25:49Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:49Z" name: success-200-isvc-b2e49-predictor-869fccf77f-86ffs.18a961847db6152c namespace: kserve-ci-e2e-test resourceVersion: "19606" uid: 7204664a-cc5c-4aa2-b2a4-276d87f5fd07 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2e49-predictor-869fccf77f namespace: kserve-ci-e2e-test resourceVersion: "18599" uid: 1f181e53-df92-4956-b329-52ea2adc3dd9 kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: 'Created pod: success-200-isvc-b2e49-predictor-869fccf77f-86ffs' metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-b2e49-predictor-869fccf77f.18a961738fc0b8af namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: 9c9ee109-2c4a-4aa8-9d1b-e5b09eba4277 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18596" uid: f5028be8-4a88-4eb6-81aa-a1f7311500fe kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Scaled up replica set success-200-isvc-b2e49-predictor-869fccf77f from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-b2e49-predictor.18a961738c1e62e2 namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: cc567256-54ff-4b48-b7b4-cb38c82292f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18598" uid: 8a32e6f6-bf6b-4421-a8f7-760461cfe6cb kind: Event lastTimestamp: "2026-04-24T19:25:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:06Z" name: success-200-isvc-b2e49-predictor.18a961770afeffdd namespace: kserve-ci-e2e-test resourceVersion: "18928" uid: 2bdd7324-8760-4469-8ffb-2591060d8c27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18598" uid: 8a32e6f6-bf6b-4421-a8f7-760461cfe6cb kind: Event lastTimestamp: "2026-04-24T19:25:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:06Z" name: success-200-isvc-b2e49-predictor.18a961770b5d28d8 namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 7a7cefbc-8f07-4485-974f-ce5dd8e852f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 8a32e6f6-bf6b-4421-a8f7-760461cfe6cb kind: Event lastTimestamp: "2026-04-24T19:25:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:36Z" name: success-200-isvc-b2e49-predictor.18a9617e09347695 namespace: kserve-ci-e2e-test resourceVersion: "19331" uid: 9484440d-2282-43a4-b42e-a3689e9ac83c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e49-predictor namespace: kserve-ci-e2e-test resourceVersion: "18811" uid: 8a32e6f6-bf6b-4421-a8f7-760461cfe6cb kind: Event lastTimestamp: "2026-04-24T19:25:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:36Z" name: success-200-isvc-b2e49-predictor.18a9617e09923419 namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: d55a3cc5-1675-4f13-85c1-521bebaaa3d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "18576" uid: 33a17291-cc66-4621-8a2b-f4b62e5ddeaa kind: Event lastTimestamp: "2026-04-24T19:25:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:24Z" name: success-200-isvc-b2e49.18a961738c21e6a4 namespace: kserve-ci-e2e-test resourceVersion: "19221" uid: 6bd5563c-b3e5-45b0-953b-da8b3ae87667 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "18576" uid: 33a17291-cc66-4621-8a2b-f4b62e5ddeaa kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-b2e49": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2e49": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-b2e49.18a961738ca8c21b namespace: kserve-ci-e2e-test resourceVersion: "18604" uid: b0b7f3d9-297e-4d9f-95ef-98127ed070ac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "18576" uid: 33a17291-cc66-4621-8a2b-f4b62e5ddeaa kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2e49": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-b2e49.18a961738ca909ab namespace: kserve-ci-e2e-test resourceVersion: "18606" uid: f291edf4-602e-459b-9320-b906e834e89d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e49 namespace: kserve-ci-e2e-test resourceVersion: "19219" uid: 33a17291-cc66-4621-8a2b-f4b62e5ddeaa kind: Event lastTimestamp: "2026-04-24T19:25:24Z" message: InferenceService [success-200-isvc-b2e49] is Ready metadata: creationTimestamp: "2026-04-24T19:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:24Z" name: success-200-isvc-b2e49.18a9617eafef5602 namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: ccde7b01-000c-4fe9-8a5f-f8acb2c04ce6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:26:27.030422Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d31ef72d3 namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: 3b84828f-6ad0-4b66-bf46-3ba9e09545a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-cb3b2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d3a1fea24 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: cdeae0af-88f1-4058-8056-bc6cc27ba6d7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d6ed173af namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 7b351c1e-80b5-4a27-9b01-4b5c4ffaf738 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d70368eae namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 685563d6-a271-4f87-8162-0ca3cbd90ede reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d73e88e34 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 1d14fb23-a808-4791-87f5-63f01e1eb0d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d745ffb27 namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 76256b95-9b11-4317-9302-401e5c67bdf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d746bee8c namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: a1c8bcc1-747a-4a8c-8a49-780698dcdb12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d78042563 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 1c5d95eb-6e9c-49af-b0ef-14a15461a2bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:26:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:26:28Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618d7890aaf0 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 0d9b249b-4a57-4eb1-bd47-b2148da45f6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:27:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:06Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9618de4203b0e namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 7974cbdf-450f-4444-a112-69f61713638c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9620e602a8bc1 namespace: kserve-ci-e2e-test resourceVersion: "24299" uid: cbc43d97-ae01-42a9-9775-b6564bc72c09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 6d5cedad-0251-4e89-96b5-e205fbec161d kind: Event lastTimestamp: "2026-04-24T19:35:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:41Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd.18a9620e602b3480 namespace: kserve-ci-e2e-test resourceVersion: "24303" uid: 95166a42-c72b-4da2-89c9-61a4b7b69ca9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cb3b2-predictor-7cb6bb55c9 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: e96f1c48-0a13-448c-aa05-2ec563937dce kind: Event lastTimestamp: "2026-04-24T19:26:27Z" message: 'Created pod: success-200-isvc-cb3b2-predictor-7cb6bb55c9-qkmvd' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-cb3b2-predictor-7cb6bb55c9.18a9618d318089f3 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: cb4f0f8a-8bd5-4bda-bef8-2e66564e783c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: a103dd4f-3861-4ec2-a3e8-0382e8365101 kind: Event lastTimestamp: "2026-04-24T19:26:26Z" message: Scaled up replica set success-200-isvc-cb3b2-predictor-7cb6bb55c9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:26Z" name: success-200-isvc-cb3b2-predictor.18a9618d2fa55573 namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: efb07aee-ff2a-42e1-8baa-bb10c00ab5f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 19c52311-183f-40b4-ae44-dbb90388dd7c kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:12Z" name: success-200-isvc-cb3b2-predictor.18a96190ae685b8e namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: cb1cd9cf-7f1b-4fda-b771-00ef5b57cc03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 19c52311-183f-40b4-ae44-dbb90388dd7c kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:12Z" name: success-200-isvc-cb3b2-predictor.18a96190aec844ef namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: aa457afa-1a53-4b92-bda0-db731938da95 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 19c52311-183f-40b4-ae44-dbb90388dd7c kind: Event lastTimestamp: "2026-04-24T19:27:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:42Z" name: success-200-isvc-cb3b2-predictor.18a961942d8bfcca namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 0efdc749-bb83-462e-9c63-08e8ef0498e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:26:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb3b2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 19c52311-183f-40b4-ae44-dbb90388dd7c kind: Event lastTimestamp: "2026-04-24T19:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:42Z" name: success-200-isvc-cb3b2-predictor.18a961942df1dba7 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 6edb248d-c9b8-43cd-8d3f-63b8384678f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 847144ef-c5e2-4b2c-9a72-ac386e065589 kind: Event lastTimestamp: "2026-04-24T19:27:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:16Z" name: success-200-isvc-cb3b2.18a9618d2f7286e8 namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 759b6c7b-260e-4c8d-8497-3dcedba11546 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 847144ef-c5e2-4b2c-9a72-ac386e065589 kind: Event lastTimestamp: "2026-04-24T19:26:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-cb3b2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cb3b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-cb3b2.18a9618d301e005c namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 65e386bd-add3-450c-983e-923714219af4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 847144ef-c5e2-4b2c-9a72-ac386e065589 kind: Event lastTimestamp: "2026-04-24T19:26:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cb3b2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:27Z" name: success-200-isvc-cb3b2.18a9618d301e595a namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: ee6abade-4737-4483-9dc4-fda46600b4bd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb3b2 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 847144ef-c5e2-4b2c-9a72-ac386e065589 kind: Event lastTimestamp: "2026-04-24T19:27:16Z" message: InferenceService [success-200-isvc-cb3b2] is Ready metadata: creationTimestamp: "2026-04-24T19:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:16Z" name: success-200-isvc-cb3b2.18a96198a020718a namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: c132a680-072b-4672-90c9-fe0d208ba628 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:15:22.019499Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d5820-predictor-5c958b5f7c-995rv to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:15:22Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f25c31ff98 namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: dd45020e-5467-4e90-9173-0950e9c6bcdc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d5820-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:22Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f26dcdc21a namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: a50e461a-a2d7-4086-9d9f-b9f394793dfe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f2993f5e8b namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: a68274d2-493a-419b-ab96-8e834518a92b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f29a9358ee namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 9b8da5d5-9a2d-4ed6-a604-23d70de1bf8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f29f7ac2ca namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 13e265da-8032-49d9-a4f5-242d46a017c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f2a0059da4 namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: baa072b5-55e6-463b-98eb-f9d465c45689 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f2a012603e namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 990c4956-fae7-4e02-b5cb-6ca9fe0e7384 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f2a302c311 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: 35e6d7e0-1aa3-4206-bbd5-fc2529428e35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:15:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:23Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f2a37b806f namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 7ee2f1ed-8b5f-4503-83a2-8a9108c38e52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:24:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:40Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a960f2d553c063 namespace: kserve-ci-e2e-test resourceVersion: "18722" uid: 882dd991-856f-415a-b143-bc0a664dfe53 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a961738c33dd2d namespace: kserve-ci-e2e-test resourceVersion: "18603" uid: b87171c3-dce9-4e09-a774-f61c36c2ce41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:36Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a961738c350b8a namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 84a6b69b-0614-4a90-912d-a9f653f4116d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: b2a3743c-17ad-490b-b19b-1434fb28d669 kind: Event lastTimestamp: "2026-04-24T19:24:40Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:40Z" name: success-200-isvc-d5820-predictor-5c958b5f7c-995rv.18a9617449b4a18a namespace: kserve-ci-e2e-test resourceVersion: "18720" uid: 1aea772f-20b9-456a-bbcd-0cc281cfb47f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d5820-predictor-5c958b5f7c namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: d290c191-5604-4bc0-90a6-54dc9ff0a61e kind: Event lastTimestamp: "2026-04-24T19:15:22Z" message: 'Created pod: success-200-isvc-d5820-predictor-5c958b5f7c-995rv' metadata: creationTimestamp: "2026-04-24T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:22Z" name: success-200-isvc-d5820-predictor-5c958b5f7c.18a960f25bb4c19a namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: b20458e1-3320-4aa8-b0dd-c6dc0e3528db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: ceb56870-e450-415c-88f1-d16afc466c3c kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: Scaled up replica set success-200-isvc-d5820-predictor-5c958b5f7c from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-d5820-predictor.18a960f25a21b875 namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 4434f4b1-a614-43e9-bdcf-f3c4abe84ac9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:15:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 9aa76000-debe-4ea5-b42a-92635caf1a2b kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:52Z" name: success-200-isvc-d5820-predictor.18a960f5d8aa7e69 namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 20057dd0-5931-4b53-a4ac-4cffdd6f1427 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:15:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 9aa76000-debe-4ea5-b42a-92635caf1a2b kind: Event lastTimestamp: "2026-04-24T19:15:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:15:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:52Z" name: success-200-isvc-d5820-predictor.18a960f5d912824e namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: eb1a300b-f456-44f8-b720-6df5da721327 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:16:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 9aa76000-debe-4ea5-b42a-92635caf1a2b kind: Event lastTimestamp: "2026-04-24T19:16:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:37Z" name: success-200-isvc-d5820-predictor.18a960fcd7184cda namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 57c95513-38d6-431a-84ba-fb7b931ab547 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:16:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d5820-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 9aa76000-debe-4ea5-b42a-92635caf1a2b kind: Event lastTimestamp: "2026-04-24T19:16:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:37Z" name: success-200-isvc-d5820-predictor.18a960fcd77f9176 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: bfa0e350-62b2-4485-a6a7-5a2e5e6045d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 820f8235-e1ba-4172-96a4-df55d5b28a47 kind: Event lastTimestamp: "2026-04-24T19:16:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:10Z" name: success-200-isvc-d5820.18a960f259fc4047 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 3bca6a19-418b-4df1-b788-58a83f68fd64 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 820f8235-e1ba-4172-96a4-df55d5b28a47 kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-d5820": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d5820": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-d5820.18a960f25a652cf9 namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: 844f41e9-5eff-4507-9f23-a79a6fdefb1a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 820f8235-e1ba-4172-96a4-df55d5b28a47 kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d5820": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:21Z" name: success-200-isvc-d5820.18a960f25a658e9c namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 9cb87fd2-5373-426f-b2b6-bbcee8617e73 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d5820 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 820f8235-e1ba-4172-96a4-df55d5b28a47 kind: Event lastTimestamp: "2026-04-24T19:16:10Z" message: InferenceService [success-200-isvc-d5820] is Ready metadata: creationTimestamp: "2026-04-24T19:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:10Z" name: success-200-isvc-d5820.18a960fd91e94739 namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 481012f2-e031-45f7-87bb-1a2045825f25 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:47.290569Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19503" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ec538-predictor-767f7c7f5-ljrct to ip-10-0-138-6.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a96183f1416763 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 7030c0c9-7a85-4015-a309-d90fc87df073 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ec538-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a9618400b05c37 namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 52b091e1-4974-448d-89a2-90465827ddd7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19523" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a961842e81894a namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 12f468c8-6927-4e26-b2ef-f8c77c766cfa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a961842fd45041 namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: fd471f4a-05d2-444b-b92d-5e29a908e702 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a9618432f95fbf namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 08e08085-959c-46dd-97fe-84b9df8923af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a96184336dbc70 namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: b98dd530-204d-47cd-b008-2e1c1a60c143 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a96184337a07b2 namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: e32fb4a4-e7d0-4c61-8210-357f8b343c4c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a9618437f8d835 namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: ad3fedf9-c3d0-42b8-9544-bef91c76f66f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:25:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:48Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a961843872f25e namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: 7a18dda3-8ec2-4085-abb6-3e3091190b16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T19:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:35:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:14Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a9618450112088 namespace: kserve-ci-e2e-test resourceVersion: "24044" uid: 15a23091-6ca5-4d9a-bc2c-e2213e233236 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a9620774ef9215 namespace: kserve-ci-e2e-test resourceVersion: "23930" uid: 3e2fcabb-042f-4c32-8f74-387a0b152c68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:35:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:12Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a9620774f06122 namespace: kserve-ci-e2e-test resourceVersion: "23934" uid: 2df5c97f-a1d1-41aa-b65f-2ed8fe83c809 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc2ae66b-d959-4e95-9f33-17fb9e2960a1 kind: Event lastTimestamp: "2026-04-24T19:35:14Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:35:14Z" name: success-200-isvc-ec538-predictor-767f7c7f5-ljrct.18a96208188cf830 namespace: kserve-ci-e2e-test resourceVersion: "24043" uid: e0858b61-7ffe-43a8-b7aa-99b3733e787e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-6.ec2.internal source: component: kubelet host: ip-10-0-138-6.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ec538-predictor-767f7c7f5 namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 1cb5659c-86bf-4362-8d54-17d7168e26f8 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Created pod: success-200-isvc-ec538-predictor-767f7c7f5-ljrct' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-ec538-predictor-767f7c7f5.18a96183f09189c2 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: f75bf1db-b7c8-4e8a-8329-5907cf5660d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: 5e4c11f7-d271-448e-b05a-434e5aa08eb2 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Scaled up replica set success-200-isvc-ec538-predictor-767f7c7f5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-ec538-predictor.18a96183ed9b2b5e namespace: kserve-ci-e2e-test resourceVersion: "19494" uid: e59060a6-0b0e-4528-8178-29dc61f6ca44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 4f43aefc-7d4a-49a1-8e57-245d2f55ac4b kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:17Z" name: success-200-isvc-ec538-predictor.18a961876c4471e6 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: f31e3f32-9ae9-41ce-ba9a-5504b9640758 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 4f43aefc-7d4a-49a1-8e57-245d2f55ac4b kind: Event lastTimestamp: "2026-04-24T19:26:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:17Z" name: success-200-isvc-ec538-predictor.18a961876ca7d590 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: c6c32edb-d27d-41bd-8443-485eff323df8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:26:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: 4f43aefc-7d4a-49a1-8e57-245d2f55ac4b kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:17Z" name: success-200-isvc-ec538-predictor.18a9618e6a04cf0d namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: f9033e82-1e6a-4f2f-b85c-81985cf2b10e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:26:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ec538-predictor namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: 4f43aefc-7d4a-49a1-8e57-245d2f55ac4b kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:17Z" name: success-200-isvc-ec538-predictor.18a9618e6a47e8fe namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: c352d707-2fc6-42df-8361-16dd585c9007 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 460405d6-0133-4a45-8047-ceebf5a8ed69 kind: Event lastTimestamp: "2026-04-24T19:26:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:45Z" name: success-200-isvc-ec538.18a96183eda00a08 namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: 05122d71-3217-4531-8872-2fe6363e3411 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 460405d6-0133-4a45-8047-ceebf5a8ed69 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-ec538": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ec538": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-ec538.18a96183ee1875b5 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: a3a3a862-c6b9-45d4-a757-d95ab3e5a060 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 460405d6-0133-4a45-8047-ceebf5a8ed69 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ec538": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:47Z" name: success-200-isvc-ec538.18a96183ee18d696 namespace: kserve-ci-e2e-test resourceVersion: "19499" uid: e20fccfa-fd72-487f-a78a-8c1b0552621e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:26:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ec538 namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: 460405d6-0133-4a45-8047-ceebf5a8ed69 kind: Event lastTimestamp: "2026-04-24T19:26:44Z" message: InferenceService [success-200-isvc-ec538] is Ready metadata: creationTimestamp: "2026-04-24T19:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:26:44Z" name: success-200-isvc-ec538.18a9619160ca0f15 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 91b64d27-fdfc-4465-9143-35a6393911bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:15:11.756071Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-48e5e-5bd9bbb6fb-bkjg2 to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:15:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:15:11Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960eff8728fe7 namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 9490c9f4-413a-43d9-ab4b-017e5e5aeaff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:12Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14030" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:12Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:15:12Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f01292c2e6 namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 4c4e82ce-c3e6-4a36-bdac-db0dfe1c1cea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48e5e} kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:12Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" metadata: creationTimestamp: "2026-04-24T19:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:12Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f013fd3e99 namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: 95559564-9eac-496c-b08b-5e9e7d868976 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48e5e} kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" in 2.051s (2.051s including waiting). Image size: 216346179 bytes.' metadata: creationTimestamp: "2026-04-24T19:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:14Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f08e432049 namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: bd6e2c6c-9ef4-4f36-ad6b-4de98aeffb63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48e5e} kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:14Z" message: 'Created container: switch-graph-48e5e' metadata: creationTimestamp: "2026-04-24T19:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:14Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f0911e87e3 namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: f0f0b8c2-9247-42de-8279-9b2abb574ced reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48e5e} kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:14Z" message: Started container switch-graph-48e5e metadata: creationTimestamp: "2026-04-24T19:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:14Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f0917b0eb3 namespace: kserve-ci-e2e-test resourceVersion: "14062" uid: 533ca097-f771-424d-99a7-25a5b9373e6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48e5e} kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:21Z" message: Stopping container switch-graph-48e5e metadata: creationTimestamp: "2026-04-24T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:21Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f252ed5a00 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 12cf0c3b-7489-473b-a417-821ae7892794 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:15:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48e5e} kind: Pod name: switch-graph-48e5e-5bd9bbb6fb-bkjg2 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 79be89e8-7b53-4d89-8b28-4ec26fb7818d kind: Event lastTimestamp: "2026-04-24T19:15:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:15:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:50Z" name: switch-graph-48e5e-5bd9bbb6fb-bkjg2.18a960f34575ec60 namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: 39b92290-73fa-4060-9b98-31e921955244 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-48e5e-5bd9bbb6fb namespace: kserve-ci-e2e-test resourceVersion: "14018" uid: e3651781-4cb0-4f8c-ae5a-687b73858665 kind: Event lastTimestamp: "2026-04-24T19:15:11Z" message: 'Created pod: switch-graph-48e5e-5bd9bbb6fb-bkjg2' metadata: creationTimestamp: "2026-04-24T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:11Z" name: switch-graph-48e5e-5bd9bbb6fb.18a960eff7f97633 namespace: kserve-ci-e2e-test resourceVersion: "14023" uid: 811a80bd-a0ec-444c-ae7c-afffea58a22d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-48e5e namespace: kserve-ci-e2e-test resourceVersion: "14016" uid: 0b915465-450e-4b16-bdcd-ca6e14cf1da5 kind: Event lastTimestamp: "2026-04-24T19:15:11Z" message: Scaled up replica set switch-graph-48e5e-5bd9bbb6fb from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:11Z" name: switch-graph-48e5e.18a960eff7120001 namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: fc55fd73-d3bd-4756-affb-44942972023c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-48e5e namespace: kserve-ci-e2e-test resourceVersion: "14032" uid: eb6cb988-6353-4c75-bd76-18c24034ee28 kind: Event lastTimestamp: "2026-04-24T19:15:11Z" message: InferenceGraph [switch-graph-48e5e] is Ready metadata: creationTimestamp: "2026-04-24T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:15:11Z" name: switch-graph-48e5e.18a960effbf18166 namespace: kserve-ci-e2e-test resourceVersion: "14034" uid: ca4af2cf-c4c9-4ad9-ae65-ba289a2071c6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:36:51.970171Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25195" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a0351-c7768db49-ftz2c to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:36:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:36:51Z" name: switch-graph-a0351-c7768db49-ftz2c.18a9621eb33f4924 namespace: kserve-ci-e2e-test resourceVersion: "25200" uid: f7f4ca5e-8d76-480b-960d-64b75c4ed06d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-a0351-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:52Z" name: switch-graph-a0351-c7768db49-ftz2c.18a9621ec3285828 namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: 061686a6-e0a8-4d20-aa53-9bcc59b4438d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25204" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:36:52Z" name: switch-graph-a0351-c7768db49-ftz2c.18a9621ef059fc67 namespace: kserve-ci-e2e-test resourceVersion: "25223" uid: 1e798cc3-73ac-4aa0-9c63-7dc82f58be7e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0351} kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:36:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:53Z" name: switch-graph-a0351-c7768db49-ftz2c.18a9621ef17fbffa namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: 46fb65ea-47c2-4053-bc7e-f2d0f7f0cbc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0351} kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:36:53Z" message: 'Created container: switch-graph-a0351' metadata: creationTimestamp: "2026-04-24T19:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:53Z" name: switch-graph-a0351-c7768db49-ftz2c.18a9621ef42ac869 namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: 30ec01d3-c1b6-44f5-9147-70cb4b14c58a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0351} kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:36:53Z" message: Started container switch-graph-a0351 metadata: creationTimestamp: "2026-04-24T19:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:36:53Z" name: switch-graph-a0351-c7768db49-ftz2c.18a9621ef499ef28 namespace: kserve-ci-e2e-test resourceVersion: "25227" uid: ed298e6a-3a4a-451e-b22f-f3f0f0ba5e60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0351} kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:53:11Z" message: Stopping container switch-graph-a0351 metadata: creationTimestamp: "2026-04-24T19:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:11Z" name: switch-graph-a0351-c7768db49-ftz2c.18a96302bc9a1cd3 namespace: kserve-ci-e2e-test resourceVersion: "32418" uid: fb3348e2-ad91-4e53-9bcc-de86194e7068 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:53:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a0351} kind: Pod name: switch-graph-a0351-c7768db49-ftz2c namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: b96571d8-c3fa-4646-96fb-1705841cc8d5 kind: Event lastTimestamp: "2026-04-24T19:53:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:53:29Z" name: switch-graph-a0351-c7768db49-ftz2c.18a96303865c0089 namespace: kserve-ci-e2e-test resourceVersion: "32626" uid: 34ab667e-3858-47f4-9f18-aaad74703ca1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a0351-c7768db49 namespace: kserve-ci-e2e-test resourceVersion: "25192" uid: 45cd61da-b114-4aff-884b-b460cc63b517 kind: Event lastTimestamp: "2026-04-24T19:36:51Z" message: 'Created pod: switch-graph-a0351-c7768db49-ftz2c' metadata: creationTimestamp: "2026-04-24T19:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:51Z" name: switch-graph-a0351-c7768db49.18a9621eb2d4b4fc namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: 0cbd2a0a-4e7d-4f86-adf9-85757220ffe2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a0351 namespace: kserve-ci-e2e-test resourceVersion: "25190" uid: ad8e08d5-72fc-4410-a912-08496d02276b kind: Event lastTimestamp: "2026-04-24T19:36:51Z" message: Scaled up replica set switch-graph-a0351-c7768db49 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:36:51Z" name: switch-graph-a0351.18a9621eb219a43f namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: 812cb552-c615-4596-b6d1-2884c03355e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:36:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a0351 namespace: kserve-ci-e2e-test resourceVersion: "25206" uid: 5fe24f23-2be2-4845-b621-240c95f02b4d kind: Event lastTimestamp: "2026-04-24T19:36:52Z" message: InferenceGraph [switch-graph-a0351] is Ready metadata: creationTimestamp: "2026-04-24T19:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:36:52Z" name: switch-graph-a0351.18a9621eb5e9deae namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: aac8c134-cec5-4e01-aa6f-dfbc47248bf5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a0351 namespace: kserve-ci-e2e-test resourceVersion: "25191" uid: 5d3ecd15-d4d1-46b5-8a69-c0fcc62bbd21 kind: Event lastTimestamp: "2026-04-24T19:37:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:21Z" name: switch-graph-a0351.18a9622231bd6b8c namespace: kserve-ci-e2e-test resourceVersion: "25472" uid: 11a7e114-fb65-4646-8f09-922317633333 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a0351 namespace: kserve-ci-e2e-test resourceVersion: "25191" uid: 5d3ecd15-d4d1-46b5-8a69-c0fcc62bbd21 kind: Event lastTimestamp: "2026-04-24T19:37:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:37:22Z" name: switch-graph-a0351.18a9622232020e62 namespace: kserve-ci-e2e-test resourceVersion: "25473" uid: 4e58ce05-edfd-47be-82ae-67886ce40461 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:16:22.090581Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d5820-6fc68fcc77-2vl8v to ip-10-0-130-85.ec2.internal metadata: creationTimestamp: "2026-04-24T19:16:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:16:22Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a9610058b5f859 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 7d841398-f3e2-4612-a1b9-77352b2e3cef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-69c5fc86d6-59q8m source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:16:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d5820-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:16:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:22Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a9610065a6812a namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: e7a225c5-302b-456d-a1cf-15df6dfc254c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:16:23Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:16:23Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a96100950becfc namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8bee3158-8a60-40f5-9db9-4fb60ffc7b10 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d5820} kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:16:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:23Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a961009636e206 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 1f3982f9-ebc9-44e3-822c-2d52d56cf4c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d5820} kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:16:23Z" message: 'Created container: switch-graph-d5820' metadata: creationTimestamp: "2026-04-24T19:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:23Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a961009a23021d namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 969c65d1-20ba-4504-9392-b869269f4b59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d5820} kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:16:23Z" message: Started container switch-graph-d5820 metadata: creationTimestamp: "2026-04-24T19:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:23Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a961009a94344d namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 1a9abf14-72af-4baa-8c50-7a734a5cb481 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d5820} kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:24:36Z" message: Stopping container switch-graph-d5820 metadata: creationTimestamp: "2026-04-24T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:36Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a9617385eccf00 namespace: kserve-ci-e2e-test resourceVersion: "18572" uid: 37901bb6-dd45-49c3-a4b6-284bedd1d7d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:24:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d5820} kind: Pod name: switch-graph-d5820-6fc68fcc77-2vl8v namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8fafa372-2edb-479c-89c2-3e6c5b16af3d kind: Event lastTimestamp: "2026-04-24T19:25:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:05Z" name: switch-graph-d5820-6fc68fcc77-2vl8v.18a961744fdaf2ed namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: d23ccb6f-9b68-4e7b-8411-bf7163bad22e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-85.ec2.internal source: component: kubelet host: ip-10-0-130-85.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d5820-6fc68fcc77 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 04ff23d3-d3b5-47d5-9eb1-7da9ad89228f kind: Event lastTimestamp: "2026-04-24T19:16:22Z" message: 'Created pod: switch-graph-d5820-6fc68fcc77-2vl8v' metadata: creationTimestamp: "2026-04-24T19:16:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:22Z" name: switch-graph-d5820-6fc68fcc77.18a96100584f42a4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 5518eb38-cbd8-4b5b-8a84-e9f834f831e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d5820 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: 83728fef-509e-4b1f-9cb5-56772e17477b kind: Event lastTimestamp: "2026-04-24T19:16:22Z" message: Scaled up replica set switch-graph-d5820-6fc68fcc77 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:16:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:22Z" name: switch-graph-d5820.18a96100579a77e0 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 50e9cc32-c7c1-463e-af94-dc5a8b695c0a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d5820 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: a2560113-b1f6-479e-8eb3-c65b14f60746 kind: Event lastTimestamp: "2026-04-24T19:16:22Z" message: InferenceGraph [switch-graph-d5820] is Ready metadata: creationTimestamp: "2026-04-24T19:16:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:22Z" name: switch-graph-d5820.18a961005cc1eb3d namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 44c18eb1-01e3-48f1-bc29-9bc5f832d5ba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:16:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d5820 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 8eab9a8a-d33d-45e3-b7bd-840bc65ce698 kind: Event lastTimestamp: "2026-04-24T19:16:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:52Z" name: switch-graph-d5820.18a96103d60561f7 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: c841f45f-dbee-430d-8542-18538f95de5f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:16:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d5820 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 8eab9a8a-d33d-45e3-b7bd-840bc65ce698 kind: Event lastTimestamp: "2026-04-24T19:16:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:16:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:52Z" name: switch-graph-d5820.18a96103d6642fbf namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 264dcb03-869d-4993-a16a-94676eb2f414 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32657"