--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:14:28.389049Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20146" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-390bf-c6ccb5b76-jqf2k to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:14:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:14:28Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d1ce7a7088 namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: aaf977c1-d301-4130-9880-42d89bd47274 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:28Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:14:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-390bf-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:28Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d1d7c0b5ce namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: 867a4336-9511-4d07-af65-bbb3cc930919 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:29Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:14:29Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:14:29Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d20b41e89d namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: c0421ffb-fc88-4862-a4fc-bf4b713827fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-390bf} kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:14:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:29Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d20c8b64c2 namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: 02ec48c5-7844-46fa-aa59-bc4b6547e675 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-390bf} kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:14:29Z" message: 'Created container: ensemble-graph-390bf' metadata: creationTimestamp: "2026-04-17T08:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:29Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d20f9532c2 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: b69f0baa-3644-4590-9139-737e045d1e86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-390bf} kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:14:29Z" message: Started container ensemble-graph-390bf metadata: creationTimestamp: "2026-04-17T08:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:29Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d2100627d7 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 19219718-c671-4c94-8078-21ef54c74e02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-390bf} kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Stopping container ensemble-graph-390bf metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d4265ebe3a namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 3eed3e12-a9cd-4d99-bcde-b575ddc8de43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-390bf} kind: Pod name: ensemble-graph-390bf-c6ccb5b76-jqf2k namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 7acd7c46-18d3-4fa4-9697-1f83492ffd81 kind: Event lastTimestamp: "2026-04-17T08:15:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:06Z" name: ensemble-graph-390bf-c6ccb5b76-jqf2k.18a716d4d3f5f3e7 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 9a0265f5-1788-442a-a0f1-841e1c55b3b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-390bf-c6ccb5b76 namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 5656b5e0-f864-4634-b031-197cc34b152b kind: Event lastTimestamp: "2026-04-17T08:14:28Z" message: 'Created pod: ensemble-graph-390bf-c6ccb5b76-jqf2k' metadata: creationTimestamp: "2026-04-17T08:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:28Z" name: ensemble-graph-390bf-c6ccb5b76.18a716d1ce112b01 namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: 2357df42-8d84-4d04-ad8e-3ae3604b206d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-390bf namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: fcd5ab1a-736c-4fde-9887-a7777ade6857 kind: Event lastTimestamp: "2026-04-17T08:14:28Z" message: Scaled up replica set ensemble-graph-390bf-c6ccb5b76 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:28Z" name: ensemble-graph-390bf.18a716d1cd32276f namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: e018a22e-c752-4ac2-898c-ac130ea46fee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-390bf namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: de539547-a27c-45e4-893f-9b69bba1ab63 kind: Event lastTimestamp: "2026-04-17T08:14:28Z" message: InferenceGraph [ensemble-graph-390bf] is Ready metadata: creationTimestamp: "2026-04-17T08:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:28Z" name: ensemble-graph-390bf.18a716d1d2169e60 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: a54130a6-8131-4d57-8468-de74f3ad9da0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:15:48.672399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-65d2e-5d4794777-nd7j8 to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:15:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:15:48Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a716e47fbd3288 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 75131731-3a3d-4be0-96b8-aebfa37bc178 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:49Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: "2026-04-17T08:15:49Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:15:49Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a716e498d6e795 namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 26cc1aab-b61e-4d25-8c30-8068a40ad2da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-65d2e} kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: "2026-04-17T08:15:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:49Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a716e49a115b2c namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 3d230176-fe05-4b30-9c68-d5d8ca9161de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-65d2e} kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: "2026-04-17T08:15:49Z" message: 'Created container: ensemble-graph-65d2e' metadata: creationTimestamp: "2026-04-17T08:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:49Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a716e49d4410c8 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: da15f532-875d-47f9-a4f9-484d6e983c5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-65d2e} kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: "2026-04-17T08:15:49Z" message: Started container ensemble-graph-65d2e metadata: creationTimestamp: "2026-04-17T08:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:49Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a716e49db31225 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 20124d5d-4246-4c7b-a1b1-5043c1c2cd87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-65d2e} kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Stopping container ensemble-graph-65d2e metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a71757ab259dde namespace: kserve-ci-e2e-test resourceVersion: "24658" uid: 799fde31-ce59-490b-9af9-6b577f00809f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:24:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-65d2e} kind: Pod name: ensemble-graph-65d2e-5d4794777-nd7j8 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: dc33d4c4-cc9f-43d5-8c27-3bee24ca8346 kind: Event lastTimestamp: "2026-04-17T08:24:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:24:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:30Z" name: ensemble-graph-65d2e-5d4794777-nd7j8.18a7175834e90503 namespace: kserve-ci-e2e-test resourceVersion: "24974" uid: 1bc826a8-6c2f-469b-9463-3582529d064f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-65d2e-5d4794777 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 3518789d-3812-4cf6-92d4-bb22a5142afc kind: Event lastTimestamp: "2026-04-17T08:15:48Z" message: 'Created pod: ensemble-graph-65d2e-5d4794777-nd7j8' metadata: creationTimestamp: "2026-04-17T08:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:48Z" name: ensemble-graph-65d2e-5d4794777.18a716e47f52d1e3 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 18fec17c-f8f9-4c39-84a8-cc9bb0851b4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-65d2e namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 7e8420a8-165d-4355-ae78-51cd38dbe2f9 kind: Event lastTimestamp: "2026-04-17T08:15:48Z" message: Scaled up replica set ensemble-graph-65d2e-5d4794777 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:48Z" name: ensemble-graph-65d2e.18a716e47ea5e29a namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 57642846-759b-4597-a2dc-072ac5b67728 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-65d2e namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 2fe0038f-98b0-49b0-a553-6f06e19c0b69 kind: Event lastTimestamp: "2026-04-17T08:15:48Z" message: InferenceGraph [ensemble-graph-65d2e] is Ready metadata: creationTimestamp: "2026-04-17T08:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:48Z" name: ensemble-graph-65d2e.18a716e4825f2cf6 namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c569ab9c-3d7c-470e-8095-75b435a7d6fa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:16:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-65d2e namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 31e415d4-dac6-46b1-87f5-ffccf4473c19 kind: Event lastTimestamp: "2026-04-17T08:16:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:18Z" name: ensemble-graph-65d2e.18a716e7fd1aa0cf namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 764d95fb-5b50-4e75-a59c-949d5030c574 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:16:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-65d2e namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 31e415d4-dac6-46b1-87f5-ffccf4473c19 kind: Event lastTimestamp: "2026-04-17T08:16:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:18Z" name: ensemble-graph-65d2e.18a716e7fd791ca3 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: e7c62ff6-de55-45d1-b338-32d6f70a9969 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:24:03.542088Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24729" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-00a8c-predictor-9b9db876f-7vskf to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a71757b8454ee1 namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: fd5e5d2c-f399-4823-80ea-16d6c87c5173 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a71757bf7193dc namespace: kserve-ci-e2e-test resourceVersion: "24748" uid: 69be08c7-875f-4a83-bcd4-5e171f0cf484 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a71757c0ad7d0e namespace: kserve-ci-e2e-test resourceVersion: "24751" uid: 4eee8e50-af08-4e35-9d4b-0f37aa331c3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a71757c53aa8b0 namespace: kserve-ci-e2e-test resourceVersion: "24752" uid: bc4b6002-9edb-44c2-8003-18161b8aaaa5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a71757c5ab4614 namespace: kserve-ci-e2e-test resourceVersion: "24753" uid: cc1094df-1b36-4f89-ace5-e705d1601f19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: "2026-04-17T08:25:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:25Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a71757eb4a6a56 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 455550bc-ed48-4f82-99b0-1763759782da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: cb684391-6aeb-41a8-a504-268ba4376434 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-00a8c-predictor-9b9db876f-7vskf.18a7176a62a5400f namespace: kserve-ci-e2e-test resourceVersion: "25901" uid: a4c0512d-8429-408d-912f-f7d7c28d8d56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-00a8c-predictor-9b9db876f namespace: kserve-ci-e2e-test resourceVersion: "24718" uid: e9c0e299-c9f2-4ea8-bf7c-3a9ee102a8a4 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'Created pod: error-404-isvc-00a8c-predictor-9b9db876f-7vskf' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor-9b9db876f.18a71757b7e16a57 namespace: kserve-ci-e2e-test resourceVersion: "24730" uid: ebfaee2f-d7b6-456f-b7f9-1a83a95d371f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24714" uid: 642b3fec-2406-4ec4-94ce-c3db31d87a63 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Scaled up replica set error-404-isvc-00a8c-predictor-9b9db876f from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c-predictor.18a71757b66646fc namespace: kserve-ci-e2e-test resourceVersion: "24722" uid: a57810b9-22f8-436b-952a-7f4386da9b41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:24:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24717" uid: 890b18b8-e24d-42dc-aa0a-6fea70c794f8 kind: Event lastTimestamp: "2026-04-17T08:25:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:18Z" name: error-404-isvc-00a8c-predictor.18a7175b350479a5 namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: 42f8b0a9-b06f-400c-8b17-f461ce0dbeba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:24:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24717" uid: 890b18b8-e24d-42dc-aa0a-6fea70c794f8 kind: Event lastTimestamp: "2026-04-17T08:25:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:18Z" name: error-404-isvc-00a8c-predictor.18a7175b35614462 namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfa0c381-c859-4c6c-93a6-46e832f9af4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: 9ed49a11-a002-465d-b777-5d27cb909d9f kind: Event lastTimestamp: "2026-04-17T08:24:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:55Z" name: error-404-isvc-00a8c.18a71757b66ed5d9 namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: e4c02136-b680-4746-ac15-19325af726b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: 9ed49a11-a002-465d-b777-5d27cb909d9f kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-00a8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-00a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c.18a71757b6f8bc4b namespace: kserve-ci-e2e-test resourceVersion: "24725" uid: 5c9ae989-abcf-4d23-87d1-a8243f415997 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: 9ed49a11-a002-465d-b777-5d27cb909d9f kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-00a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-00a8c.18a71757b6f92fd2 namespace: kserve-ci-e2e-test resourceVersion: "24728" uid: 85f29de6-8049-4576-90c7-7d5566dd3ab1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "25311" uid: 9ed49a11-a002-465d-b777-5d27cb909d9f kind: Event lastTimestamp: "2026-04-17T08:24:55Z" message: InferenceService [error-404-isvc-00a8c] is Ready metadata: creationTimestamp: "2026-04-17T08:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:55Z" name: error-404-isvc-00a8c.18a71763ceeb4a15 namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: a80e7d9c-3bf4-43bb-9c22-c171cc2b4a5e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:02:19.172481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-08159-predictor-6c4676d8f5-mkkbs to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:02:19Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7162805c8cebd namespace: kserve-ci-e2e-test resourceVersion: "13932" uid: 3dca5816-e9b5-4f2e-8dae-fd226d80afcf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13941" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:02:19Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a716280f020cfd namespace: kserve-ci-e2e-test resourceVersion: "13964" uid: 462ac8e3-700b-4d92-8d91-9056d8643bb1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:19Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7162810493778 namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: c25e465d-a935-44b6-9239-196489381eb0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:02:32Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" in 13.152s (13.152s including waiting). Image size: 1332301412 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:32Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7162b203c62fa namespace: kserve-ci-e2e-test resourceVersion: "14140" uid: 09feaf1e-19a1-485e-b450-ab0bc8faf677 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:02:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:32Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7162b26366b1f namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: 766460da-dd76-4707-91bf-418d788c3422 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:02:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:32Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7162b26b213a6 namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 8f80bd75-c373-4fac-8101-276e78912aa1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7162b2a7861aa namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 1fdc283e-6ccd-4cab-9eb4-5f04e14c7261 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: c14a4091-fd8e-410c-a9bc-8a3dd336f24f kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs.18a7163df54cfffd namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 3e1ef7bd-8266-493b-b2c4-3843350be58e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-08159-predictor-6c4676d8f5 namespace: kserve-ci-e2e-test resourceVersion: "13916" uid: a4da8d8c-a484-4d47-894d-0a388f4d1b2b kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: 'Created pod: error-404-isvc-08159-predictor-6c4676d8f5-mkkbs' metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: error-404-isvc-08159-predictor-6c4676d8f5.18a71628054c93b7 namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: 2e1e8fb9-a2d2-4bcc-aaed-209cf2f1bfda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "13914" uid: 3cfa7ac4-e622-42fd-bf81-f671a126f1a7 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Scaled up replica set error-404-isvc-08159-predictor-6c4676d8f5 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: error-404-isvc-08159-predictor.18a716280418a2fb namespace: kserve-ci-e2e-test resourceVersion: "13918" uid: b2a08f72-05ef-474d-8614-7acb174864eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "13915" uid: 83a65fd5-82a2-4075-8c45-3fed226ba186 kind: Event lastTimestamp: "2026-04-17T08:02:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:34Z" name: error-404-isvc-08159-predictor.18a7162b82a43ab3 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 0b9fa1ba-10f8-4fdd-b12f-754a72be01f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "13915" uid: 83a65fd5-82a2-4075-8c45-3fed226ba186 kind: Event lastTimestamp: "2026-04-17T08:02:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:34Z" name: error-404-isvc-08159-predictor.18a7162b83dc7f30 namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 583ec2e1-8eae-4350-9d81-22dccdcd1931 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:02:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: 83a65fd5-82a2-4075-8c45-3fed226ba186 kind: Event lastTimestamp: "2026-04-17T08:03:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:49Z" name: error-404-isvc-08159-predictor.18a7162f027eabe7 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 25c0c441-4cb0-493e-b30f-4d5813ec44af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:02:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: 83a65fd5-82a2-4075-8c45-3fed226ba186 kind: Event lastTimestamp: "2026-04-17T08:03:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:49Z" name: error-404-isvc-08159-predictor.18a7162f02e532ce namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 6f99772a-2462-42e3-9dfc-3c58d47f1ef5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08159 namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: 9432079b-58f1-438f-bba5-3814f520d2f6 kind: Event lastTimestamp: "2026-04-17T08:03:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:33Z" name: error-404-isvc-08159.18a7162803f511ed namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 7dda5d1d-e9a1-4ac8-8497-4c2ea103beed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08159 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 9432079b-58f1-438f-bba5-3814f520d2f6 kind: Event lastTimestamp: "2026-04-17T08:03:33Z" message: InferenceService [error-404-isvc-08159] is Ready metadata: creationTimestamp: "2026-04-17T08:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:33Z" name: error-404-isvc-08159.18a716396257c2bf namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 6e70d935-012e-4cd0-943a-6dec173f6773 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:25:23.843422Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-32e35-predictor-bc6967c69-44xft to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a7176a6a9a6ea8 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 887eaabb-69e9-4c92-a5e9-343f519465ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a7176a722d3656 namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: c209e681-3397-475e-9ee9-0ceae22639d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a7176a7357e6ae namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 4f641421-4285-4235-a192-7c6139738fde reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: "2026-04-17T08:25:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:24Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a7176a777202ac namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 8078bccc-9c1c-4ca6-85e3-d721e30ba8de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: "2026-04-17T08:25:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:24Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a7176a77e507a1 namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 96706370-2636-4df9-ac79-4dfe7a6b1b83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: "2026-04-17T08:26:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:26:05Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a7176a99b2a647 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: fe392431-e704-493b-8e7b-32c46a4e784a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:34:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32e35-predictor-bc6967c69-44xft namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: d4de8432-25db-4204-8330-4c9c151e2824 kind: Event lastTimestamp: "2026-04-17T08:34:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:34:48Z" name: error-404-isvc-32e35-predictor-bc6967c69-44xft.18a717eded5118b8 namespace: kserve-ci-e2e-test resourceVersion: "30124" uid: 5a49b1a3-51ae-4bd0-9dc3-451ec41ff9ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-32e35-predictor-bc6967c69 namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: 962757f9-4b7f-47bd-b7b2-a4cb83869e8b kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: 'Created pod: error-404-isvc-32e35-predictor-bc6967c69-44xft' metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35-predictor-bc6967c69.18a7176a6a3b522b namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: d17ad926-41e6-4beb-9dee-fa1971c56da3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 8698e23d-6026-495b-9682-76faa52cb69d kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Scaled up replica set error-404-isvc-32e35-predictor-bc6967c69 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35-predictor.18a7176a68b2e5b0 namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: 98437582-6ce5-4afc-877e-2fad217f2f79 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:25:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: b85520b6-3844-4517-a0ab-6ffdd3287908 kind: Event lastTimestamp: "2026-04-17T08:25:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:53Z" name: error-404-isvc-32e35-predictor.18a7176de74e6d84 namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 5a4c3b49-8f85-48cc-9fdb-2e3dbb15d5a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:25:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: b85520b6-3844-4517-a0ab-6ffdd3287908 kind: Event lastTimestamp: "2026-04-17T08:25:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:53Z" name: error-404-isvc-32e35-predictor.18a7176de78c7600 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: 9be21793-a250-4417-aee4-c5c6ead393b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:26:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: b85520b6-3844-4517-a0ab-6ffdd3287908 kind: Event lastTimestamp: "2026-04-17T08:27:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:27:08Z" name: error-404-isvc-32e35-predictor.18a71774e4dc6684 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 0963b6a3-978c-4f50-80f4-425bc234ef9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:26:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: b85520b6-3844-4517-a0ab-6ffdd3287908 kind: Event lastTimestamp: "2026-04-17T08:27:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:27:08Z" name: error-404-isvc-32e35-predictor.18a71774e53fa2ce namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: fbedc8f7-2dda-491a-a0c2-0ed6196c2c31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e35 namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 4b0758d3-8679-4abd-84b9-da3e9c2f7166 kind: Event lastTimestamp: "2026-04-17T08:26:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:15Z" name: error-404-isvc-32e35.18a7176a68cf76ba namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 52368f5b-e3ae-4e16-bdd7-54976ba47aa6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e35 namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 4b0758d3-8679-4abd-84b9-da3e9c2f7166 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-32e35": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-32e35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35.18a7176a69527a6b namespace: kserve-ci-e2e-test resourceVersion: "25950" uid: 28982d93-8b08-41de-a197-4a36931d05dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e35 namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 4b0758d3-8679-4abd-84b9-da3e9c2f7166 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-32e35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:23Z" name: error-404-isvc-32e35.18a7176a6952c662 namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 18880a83-8d3f-4d38-992c-d62978c856fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 4b0758d3-8679-4abd-84b9-da3e9c2f7166 kind: Event lastTimestamp: "2026-04-17T08:26:15Z" message: InferenceService [error-404-isvc-32e35] is Ready metadata: creationTimestamp: "2026-04-17T08:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:15Z" name: error-404-isvc-32e35.18a717767db108a7 namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: 917664e9-8e73-4d2f-bdd9-c34b90215fc9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:13:54.169252Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-375fe-predictor-85794dd7c-zw8vd to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716c9d6d1cf62 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 8cb4a180-4a22-42f3-8f99-6b9c66e0a8e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716c9ddf3241d namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 17a9f726-c028-4238-a60b-ef6a84ced409 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716c9df042a33 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: bf6c7947-6819-4438-b55e-1b916cc0c4c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716c9e24a370b namespace: kserve-ci-e2e-test resourceVersion: "19826" uid: d1e72ae9-e111-4a26-9d74-2605edeb4740 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716c9e2baeada namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 0515e8da-5b89-4c8d-973a-ee3897009609 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: "2026-04-17T08:15:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:15Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716c9f4943bfe namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: ecc0db6e-342b-466e-87c7-5f3ead17e868 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 0180cef1-5b9f-4eed-b1e8-8060286f7ac7 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-375fe-predictor-85794dd7c-zw8vd.18a716dc7eaf6f69 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 8dc4186c-1b09-4322-bd66-edef283c559f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-375fe-predictor-85794dd7c namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 0d20e153-e806-43d0-b97a-f789d585604e kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'Created pod: error-404-isvc-375fe-predictor-85794dd7c-zw8vd' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor-85794dd7c.18a716c9d6580ff7 namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: b49404e4-3d51-4a4c-8370-331aa46b973d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: 3f59c42d-8940-4c85-8761-0285ce469169 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Scaled up replica set error-404-isvc-375fe-predictor-85794dd7c from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe-predictor.18a716c9d5398ff6 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: 9592b1b5-1285-4d26-bbb9-f9b6882ba07f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 8a8a8629-d487-4991-84d3-2331002cabdd kind: Event lastTimestamp: "2026-04-17T08:14:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:09Z" name: error-404-isvc-375fe-predictor.18a716cd53ac99ce namespace: kserve-ci-e2e-test resourceVersion: "19966" uid: 3755d1e6-7c1c-417e-aeb2-5d7944969e3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 8a8a8629-d487-4991-84d3-2331002cabdd kind: Event lastTimestamp: "2026-04-17T08:14:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:09Z" name: error-404-isvc-375fe-predictor.18a716cd540ef8eb namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 1dcccbe2-0995-41ba-962e-1f80922b4437 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:14:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 8a8a8629-d487-4991-84d3-2331002cabdd kind: Event lastTimestamp: "2026-04-17T08:15:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:09Z" name: error-404-isvc-375fe-predictor.18a716d0d2a7e1ef namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 5ebdc433-c459-49b0-8a8d-b315ae8a46e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:14:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 8a8a8629-d487-4991-84d3-2331002cabdd kind: Event lastTimestamp: "2026-04-17T08:15:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:09Z" name: error-404-isvc-375fe-predictor.18a716d0d3075ff5 namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: fe49f97d-568b-483f-a9df-71d3d57a8e78 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: d036770e-f0d3-406b-a849-a05f1ac12225 kind: Event lastTimestamp: "2026-04-17T08:14:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:45Z" name: error-404-isvc-375fe.18a716c9d51a5949 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: ed51ae27-85aa-4c08-88cf-0ad75f45717b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: d036770e-f0d3-406b-a849-a05f1ac12225 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-375fe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-375fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe.18a716c9d58daaed namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 7507997c-a74a-4457-a379-bf8bd172f188 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: d036770e-f0d3-406b-a849-a05f1ac12225 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-375fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-375fe.18a716c9d58df319 namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: d6d570ce-8a47-4136-abcd-6bb250e1d641 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: d036770e-f0d3-406b-a849-a05f1ac12225 kind: Event lastTimestamp: "2026-04-17T08:14:45Z" message: InferenceService [error-404-isvc-375fe] is Ready metadata: creationTimestamp: "2026-04-17T08:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:45Z" name: error-404-isvc-375fe.18a716d5d80d3405 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 26c0d4d2-d96a-4550-abe2-d73ce808157c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:13:18.353210Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-390bf-predictor-bd86b758f-cd2fh to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716c1800459d4 namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: 0af62ddb-0bf5-4bde-ab39-eda583b8288f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716c187bfd45c namespace: kserve-ci-e2e-test resourceVersion: "19426" uid: 3a62d99a-1eb3-4364-a11e-79c9c95333b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716c188f08580 namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: cfd3c4e2-d56f-49d7-a854-876ffa7bf781 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716c18c83633d namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: f6be789f-48cb-4004-805e-6a618c645a42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716c18d27b148 namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: 14ab7384-82e8-4c6f-91df-a8896b231f83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: "2026-04-17T08:14:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:40Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716c1c7c36df0 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: bdf9f2a5-4a78-45e9-b551-d3bd2f4be923 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 241f9031-3f12-4b32-9353-ccb3e1a055d6 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-390bf-predictor-bd86b758f-cd2fh.18a716d432689491 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 19c46ca2-ae84-453e-afc3-4deb7f0e7d2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-390bf-predictor-bd86b758f namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 85efe4a2-e112-4728-ad8a-5e0c2dcff982 kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'Created pod: error-404-isvc-390bf-predictor-bd86b758f-cd2fh' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor-bd86b758f.18a716c17fa013bd namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: 66d93ded-c7a7-43a5-81cd-3db7c6649440 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-390bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: e3883f9e-06e6-4c45-8ca4-14770afdaa7a kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Scaled up replica set error-404-isvc-390bf-predictor-bd86b758f from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf-predictor.18a716c17dd8a4bf namespace: kserve-ci-e2e-test resourceVersion: "19399" uid: cd977a56-9150-4c88-aa72-df760002c0dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: ee5711f3-d35d-421c-a5a6-e5e0b82f3ae6 kind: Event lastTimestamp: "2026-04-17T08:14:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:18Z" name: error-404-isvc-390bf-predictor.18a716c4fc3e7cd2 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: 81436abb-2a02-40cc-b6ce-1f05309d5be8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: ee5711f3-d35d-421c-a5a6-e5e0b82f3ae6 kind: Event lastTimestamp: "2026-04-17T08:14:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:18Z" name: error-404-isvc-390bf-predictor.18a716c4fc9b5321 namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 33655a97-a38f-499d-bef6-324767078e49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: 2c4f5ee5-5b66-46c9-a9c8-92156875a71c kind: Event lastTimestamp: "2026-04-17T08:14:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:10Z" name: error-404-isvc-390bf.18a716c17daf4541 namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: 22ec2eae-5f16-4c3f-a6cc-fa54942e45ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: 2c4f5ee5-5b66-46c9-a9c8-92156875a71c kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-390bf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-390bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf.18a716c17dfa283c namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 90281dfa-8c60-4a5e-a767-31414e8c897a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19370" uid: 2c4f5ee5-5b66-46c9-a9c8-92156875a71c kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-390bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-390bf.18a716c17dfa7331 namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: f00db408-b867-4df8-ba48-7ad115838224 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 2c4f5ee5-5b66-46c9-a9c8-92156875a71c kind: Event lastTimestamp: "2026-04-17T08:14:10Z" message: InferenceService [error-404-isvc-390bf] is Ready metadata: creationTimestamp: "2026-04-17T08:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:10Z" name: error-404-isvc-390bf.18a716cdab794ada namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 15ba4861-753c-4760-b286-8c29ac9e8279 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:03:53.400686Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a7163df638f55a namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: bd3c7925-baf8-4d27-8800-8c2d4ce45876 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a7163dfd982e8c namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 14037f4c-df20-4f9d-81e3-abe43b0a81fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a7163dfebc301e namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 6bf149cc-6ef8-491f-90c1-6e960638923d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a7163e05ab7149 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 5f45e634-ce3d-4d21-b5f1-d9036e062f9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a7163e06264c4e namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 98884e7a-43f0-48ad-ae6b-a1e3f05861d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: "2026-04-17T08:04:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:34Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a7163e160abee1 namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: c5a5d873-a514-4aff-b515-35c4f6e10d11 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: cae50485-7a94-4576-8aa1-6fd93af26606 kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx.18a716c17cd6a1f6 namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: d5a87f53-7340-4186-89ef-fc90352dbabb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4828d-predictor-5cbf5bc797 namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: ea3ad397-818c-45b6-9eda-52ddfe52a757 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Created pod: error-404-isvc-4828d-predictor-5cbf5bc797-n6xzx' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor-5cbf5bc797.18a7163df5e3dd6e namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 4fb387a5-132f-4eb1-9c97-ea1a8b08446b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 7f8ffc3f-671a-48bb-a13e-2621b23d0a15 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Scaled up replica set error-404-isvc-4828d-predictor-5cbf5bc797 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d-predictor.18a7163df463ff4d namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: f5e41857-d0f6-4b30-aee1-bbf5192368f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 752658cb-4dd1-4314-ba92-9612bb9aec14 kind: Event lastTimestamp: "2026-04-17T08:04:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:08Z" name: error-404-isvc-4828d-predictor.18a7164172d424b6 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: 015b9316-4fa0-424c-a7eb-077a94924f98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 752658cb-4dd1-4314-ba92-9612bb9aec14 kind: Event lastTimestamp: "2026-04-17T08:04:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:08Z" name: error-404-isvc-4828d-predictor.18a716417335016b namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: 6f7b2fae-c8d1-4591-84af-7f3a166b5642 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 752658cb-4dd1-4314-ba92-9612bb9aec14 kind: Event lastTimestamp: "2026-04-17T08:05:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:08Z" name: error-404-isvc-4828d-predictor.18a71644f1dbc37c namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 7efc44d6-5902-43d7-8410-ce6416cb33bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 752658cb-4dd1-4314-ba92-9612bb9aec14 kind: Event lastTimestamp: "2026-04-17T08:05:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:08Z" name: error-404-isvc-4828d-predictor.18a71644f23d2d2c namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: 17b7a34c-3245-4447-baae-0e7132ae2911 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 04234860-9c39-4590-bc8b-68700cd106a0 kind: Event lastTimestamp: "2026-04-17T08:04:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:45Z" name: error-404-isvc-4828d.18a7163df44048a9 namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: 95f3cba9-f41b-4ab5-9725-69126a6ebbb1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 04234860-9c39-4590-bc8b-68700cd106a0 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Failed to update status for InferenceService "error-404-isvc-4828d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4828d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d.18a7163df4cf7a44 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: ca73cf41-09f0-481c-a4c6-15370b703c9f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 04234860-9c39-4590-bc8b-68700cd106a0 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4828d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:53Z" name: error-404-isvc-4828d.18a7163df4cfc62e namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: f609120d-6af1-41a2-a7a2-e8444239a9c5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: 04234860-9c39-4590-bc8b-68700cd106a0 kind: Event lastTimestamp: "2026-04-17T08:04:44Z" message: InferenceService [error-404-isvc-4828d] is Ready metadata: creationTimestamp: "2026-04-17T08:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:44Z" name: error-404-isvc-4828d.18a71649f99e6b6c namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 2e84817c-4c49-4e78-b62a-b6aa1196916d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:24:39.382273Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a71760108324d7 namespace: kserve-ci-e2e-test resourceVersion: "25136" uid: 35ef2451-1c4c-4db4-b9d6-aeec6cc55708 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25142" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a7176017f52944 namespace: kserve-ci-e2e-test resourceVersion: "25152" uid: 4369c95e-5f1c-43c9-a057-979e5d0db8ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a717601936a32e namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: 75103040-43ba-4cf2-8be2-603f69b9b495 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a717601ca10468 namespace: kserve-ci-e2e-test resourceVersion: "25158" uid: 8b6bfeef-9d4b-4567-9514-38673726adf7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a717601d117091 namespace: kserve-ci-e2e-test resourceVersion: "25159" uid: 22232f4a-b4ad-4b3f-8dce-602fa02c72cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:24:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: "2026-04-17T08:42:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:24:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:42:11Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a717605352c606 namespace: kserve-ci-e2e-test resourceVersion: "33232" uid: f9b53082-d373-4cba-8581-821afdfd511d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: b9868b43-3967-4ab1-aa78-76b17d95df70 kind: Event lastTimestamp: "2026-04-17T08:42:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:42:08Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77.18a718546b721ede namespace: kserve-ci-e2e-test resourceVersion: "33183" uid: 493933b1-3019-4dca-bbad-c0aa7e54cbd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-54cf7-predictor-7bbd5dfff9 namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: de66b577-85e1-4ac0-9247-f5ea21ca47e1 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'Created pod: error-404-isvc-54cf7-predictor-7bbd5dfff9-qxt77' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor-7bbd5dfff9.18a71760101a05a7 namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: 2b922719-ecc2-46aa-b3de-b67f1c8e3556 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: 1b6d6661-baad-4f64-8c3f-5cff00e32022 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Scaled up replica set error-404-isvc-54cf7-predictor-7bbd5dfff9 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7-predictor.18a717600eb61b65 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: 6d7ab1c2-d992-4436-8f2d-839a2c6cb13b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 0456cd44-ca14-430c-8c6d-369049a195b8 kind: Event lastTimestamp: "2026-04-17T08:24:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:54Z" name: error-404-isvc-54cf7-predictor.18a717638d22ffcd namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: c7fcfcb3-aa9d-4902-ad69-ea3949994dc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 0456cd44-ca14-430c-8c6d-369049a195b8 kind: Event lastTimestamp: "2026-04-17T08:24:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:54Z" name: error-404-isvc-54cf7-predictor.18a717638d8115b9 namespace: kserve-ci-e2e-test resourceVersion: "25297" uid: 5929fa80-7f72-4431-b905-6b839329aff8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:25:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25296" uid: 0456cd44-ca14-430c-8c6d-369049a195b8 kind: Event lastTimestamp: "2026-04-17T08:25:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:54Z" name: error-404-isvc-54cf7-predictor.18a717670c25b64a namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: 03949f09-9932-4165-8528-375dc460b5eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:25:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25296" uid: 0456cd44-ca14-430c-8c6d-369049a195b8 kind: Event lastTimestamp: "2026-04-17T08:25:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:54Z" name: error-404-isvc-54cf7-predictor.18a717670c82d3d0 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 9b5c6b9a-4447-4241-90b3-b532fdbfae69 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "25107" uid: 925758ec-f1a9-4f01-80ed-7fca1cfcdef0 kind: Event lastTimestamp: "2026-04-17T08:25:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:31Z" name: error-404-isvc-54cf7.18a717600e9657eb namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: aa0f87c0-4016-48f7-b9b5-2cc381f1f22f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "25107" uid: 925758ec-f1a9-4f01-80ed-7fca1cfcdef0 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-54cf7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-54cf7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7.18a717600ee1d016 namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: f68cc525-d420-4465-b232-a1390035631e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "25107" uid: 925758ec-f1a9-4f01-80ed-7fca1cfcdef0 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-54cf7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-54cf7.18a717600ee228e9 namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: dd68e31d-67e7-4e91-add6-c27a07ea6f25 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "26078" uid: 925758ec-f1a9-4f01-80ed-7fca1cfcdef0 kind: Event lastTimestamp: "2026-04-17T08:25:31Z" message: InferenceService [error-404-isvc-54cf7] is Ready metadata: creationTimestamp: "2026-04-17T08:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:31Z" name: error-404-isvc-54cf7.18a7176c36c066fe namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ea34993c-5e30-4eb6-a77b-c3adeca667b6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:15:14.474123Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-60721-predictor-85469d7b4f-mhw92 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a716dc895cd1d2 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 84525621-0c9c-44cf-8846-ff2a9819248f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a716dc9095a25d namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: b068cd03-70ad-4b58-8f7d-92ae35808774 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a716dc91be0ba1 namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 55fca870-abe2-4dd3-8b90-6bece8e8a4fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a716dc950b986a namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 96876456-c951-4897-ad57-5232b7454b04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a716dc95864413 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 3974b8a7-923a-4f9c-a0f0-f707836b4abe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: "2026-04-17T08:15:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:55Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a716dca2f8a0de namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: 99319eab-846a-4bd7-aca9-5ed0ac3962e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60721-predictor-85469d7b4f-mhw92 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 51a08f91-43d7-44f2-ac02-5f1951a3fb98 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: error-404-isvc-60721-predictor-85469d7b4f-mhw92.18a7176007d812c9 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: df524749-7120-479d-a1e6-c2fa488ad625 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-60721-predictor-85469d7b4f namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: dc2ab4c0-cd87-4464-a83e-8dda18f41138 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: 'Created pod: error-404-isvc-60721-predictor-85469d7b4f-mhw92' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor-85469d7b4f.18a716dc8903d985 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: ff83dba5-d0ab-40ad-8557-11f81933adf0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: 6d40dc1f-685c-476e-a72b-1c43c6f9bd8b kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Scaled up replica set error-404-isvc-60721-predictor-85469d7b4f from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721-predictor.18a716dc872ec321 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 44eb9948-9a2d-4e64-b9df-9d82d9badfee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: c8a73d05-3d2b-4d56-8034-3e37ed6387e2 kind: Event lastTimestamp: "2026-04-17T08:15:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:29Z" name: error-404-isvc-60721-predictor.18a716e0055859f3 namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 40fcaa7e-43bd-4a81-90a3-ca3a16f5beff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: c8a73d05-3d2b-4d56-8034-3e37ed6387e2 kind: Event lastTimestamp: "2026-04-17T08:15:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:29Z" name: error-404-isvc-60721-predictor.18a716e005b58fd2 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: ed6c7e4a-94b9-4e5d-9e77-2cd2744f4676 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:15:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: c8a73d05-3d2b-4d56-8034-3e37ed6387e2 kind: Event lastTimestamp: "2026-04-17T08:16:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:44Z" name: error-404-isvc-60721-predictor.18a716e384597b7f namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: 47fe75ec-328c-453b-9f17-5361288eff17 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:15:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: c8a73d05-3d2b-4d56-8034-3e37ed6387e2 kind: Event lastTimestamp: "2026-04-17T08:16:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:44Z" name: error-404-isvc-60721-predictor.18a716e384b50dc6 namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 8d640645-0b6b-4810-b9f3-994203cb9080 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60721 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 931b558e-6acf-467d-9b7e-b0e694ab62ce kind: Event lastTimestamp: "2026-04-17T08:16:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:05Z" name: error-404-isvc-60721.18a716dc86ab0479 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: c3036e59-9423-4480-850c-590629e1ae2b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60721 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 931b558e-6acf-467d-9b7e-b0e694ab62ce kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: 'Failed to update status for InferenceService "error-404-isvc-60721": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60721": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721.18a716dc8748b474 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 1c6ce2f3-7550-4639-9a2b-0563b18d8d0f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60721 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 931b558e-6acf-467d-9b7e-b0e694ab62ce kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60721": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:14Z" name: error-404-isvc-60721.18a716dc8748fa02 namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: 13eb323e-fdb2-4689-ad14-1621d1a7c392 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60721 namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: 931b558e-6acf-467d-9b7e-b0e694ab62ce kind: Event lastTimestamp: "2026-04-17T08:16:05Z" message: InferenceService [error-404-isvc-60721] is Ready metadata: creationTimestamp: "2026-04-17T08:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:05Z" name: error-404-isvc-60721.18a716e88693d844 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: e4bb68be-ec7c-441f-99e1-17c6c3ec9bb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:14:38.629932Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-65d2e-predictor-8468db4849-mbmht to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a716d430e1e0b5 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 372bbe8f-0c13-4b1c-a1ff-d750567bec1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a716d438357f01 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: f873a0f4-4b77-4a95-82fb-f2954a741839 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a716d439614a48 namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 640a60c8-5820-4011-9d39-1e24226244b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a716d43e9c8f16 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: c7c22aa2-020a-4a24-84a1-8160aec774b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a716d43f18d38d namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: d0a162eb-dc97-4b91-8c4f-080cf0715929 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:14:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: "2026-04-17T08:15:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:20Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a716d47712c735 namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: ac649ebb-eef8-4436-bbce-5ca0ecd59ecb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65d2e-predictor-8468db4849-mbmht namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 1fe9f135-f6a5-41f7-80e1-669507d408ce kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: error-404-isvc-65d2e-predictor-8468db4849-mbmht.18a71757b5f40a8d namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: 0164cb6d-cb38-4a39-8a64-b5eba54264ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-65d2e-predictor-8468db4849 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 289ac847-4136-4dca-afa9-e8aa549fac61 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: 'Created pod: error-404-isvc-65d2e-predictor-8468db4849-mbmht' metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor-8468db4849.18a716d430800f0b namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 5a7630a9-9f7e-4f79-be35-4ee0928d65f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 53d3b4f6-db5c-4826-bf24-e4125e03d703 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Scaled up replica set error-404-isvc-65d2e-predictor-8468db4849 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:38Z" name: error-404-isvc-65d2e-predictor.18a716d42f069c0a namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 08656757-576f-4484-b8ad-d74810fda520 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:14:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 6ca24eb3-336e-4d7b-8ab4-d001176f59af kind: Event lastTimestamp: "2026-04-17T08:15:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:08Z" name: error-404-isvc-65d2e-predictor.18a716d7ad7767ab namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: dbf9913d-4224-4074-afd9-a8f5a4d906b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:14:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 6ca24eb3-336e-4d7b-8ab4-d001176f59af kind: Event lastTimestamp: "2026-04-17T08:15:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:08Z" name: error-404-isvc-65d2e-predictor.18a716d7add72e5f namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: f40803db-8875-4c7a-af5a-967984b6f6e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:15:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 6ca24eb3-336e-4d7b-8ab4-d001176f59af kind: Event lastTimestamp: "2026-04-17T08:15:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:53Z" name: error-404-isvc-65d2e-predictor.18a716deab2a198b namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: d90e9392-896e-4e77-ab66-e3ead43fe096 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:15:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 6ca24eb3-336e-4d7b-8ab4-d001176f59af kind: Event lastTimestamp: "2026-04-17T08:15:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:53Z" name: error-404-isvc-65d2e-predictor.18a716deab8805ba namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: e41f4793-1e6f-4144-92ef-f798e49db76e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65d2e namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 83edf250-435c-488b-b405-8fb40876c469 kind: Event lastTimestamp: "2026-04-17T08:15:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:30Z" name: error-404-isvc-65d2e.18a716d42ee52a98 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 7f825ec3-8e01-4e76-8b38-7ba6e6e8e5b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65d2e namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 83edf250-435c-488b-b405-8fb40876c469 kind: Event lastTimestamp: "2026-04-17T08:15:30Z" message: InferenceService [error-404-isvc-65d2e] is Ready metadata: creationTimestamp: "2026-04-17T08:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:30Z" name: error-404-isvc-65d2e.18a716e05ab91a35 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: c402d057-8462-4cb8-bb17-79116aa88034 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:04:29.274234Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3bdf-predictor-847869ccdf-76d9k to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a716465073de7e namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 34d7a059-6cd9-4d60-a483-ef40b27bfc90 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a7164658875377 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 04d5883d-6043-444a-b1ca-711fd319747b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a7164659c7e3a6 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 84f4efd4-f6ea-46d3-b641-991c0b4cdc26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a71646618c4f04 namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 077600f2-564e-4009-8e37-a2f838612758 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a71646621811c1 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: edaf3bb0-e3dc-41cf-a81e-9a703b122012 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: "2026-04-17T08:05:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:11Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a716467e582829 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: c4cd4b9a-d466-41f7-9e01-fff4db3b52e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 4189fa80-801e-4640-a9a4-68c294737e2d kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k.18a716c9d4abb106 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: ea8e407e-e919-4496-9f7e-cafa31abdbcb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3bdf-predictor-847869ccdf namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 82744ad7-8119-4156-8724-e5c5c2de71c2 kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: 'Created pod: error-404-isvc-c3bdf-predictor-847869ccdf-76d9k' metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor-847869ccdf.18a716464fe80128 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: fec4add6-82bb-432f-b08f-be0ec0ce0299 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: f61c5c4d-7288-49fa-8801-80b41b3c02ce kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Scaled up replica set error-404-isvc-c3bdf-predictor-847869ccdf from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:29Z" name: error-404-isvc-c3bdf-predictor.18a716464e3c1d3d namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 7c2f2cf1-9dea-4efe-a6bf-616046f335e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 5a182564-7eff-4f61-a5e9-6b639f12ee94 kind: Event lastTimestamp: "2026-04-17T08:04:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:44Z" name: error-404-isvc-c3bdf-predictor.18a71649cc81d10e namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 68f638a1-49d2-4ec7-b2d6-60a989abc2a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 5a182564-7eff-4f61-a5e9-6b639f12ee94 kind: Event lastTimestamp: "2026-04-17T08:04:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:44Z" name: error-404-isvc-c3bdf-predictor.18a71649ccdd781c namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b74a5d9f-75e8-4c7e-bd14-17be34a266b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:04:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 5a182564-7eff-4f61-a5e9-6b639f12ee94 kind: Event lastTimestamp: "2026-04-17T08:05:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:44Z" name: error-404-isvc-c3bdf-predictor.18a7164d4b9b04cc namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: e362676e-7af3-4c41-9368-64a793d40c27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:04:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 5a182564-7eff-4f61-a5e9-6b639f12ee94 kind: Event lastTimestamp: "2026-04-17T08:05:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:44Z" name: error-404-isvc-c3bdf-predictor.18a7164d4bdd76c0 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 877d2f3e-764a-4f04-9218-f26a7a0d0b76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: 8696fa19-10ac-4f0b-9b5c-fd8f2a17b7d9 kind: Event lastTimestamp: "2026-04-17T08:05:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:05:21Z" name: error-404-isvc-c3bdf.18a716464df954ad namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: fcb5389d-5b3f-47c1-8982-0b335c57b415 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 8696fa19-10ac-4f0b-9b5c-fd8f2a17b7d9 kind: Event lastTimestamp: "2026-04-17T08:05:21Z" message: InferenceService [error-404-isvc-c3bdf] is Ready metadata: creationTimestamp: "2026-04-17T08:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:05:21Z" name: error-404-isvc-c3bdf.18a7165261d52478 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 885b07e6-9d0a-44b4-bc08-aaeb48c9a573 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:02:18.903699Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13851" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-69d5d56664-444sz to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:02:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:02:18Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a71627f5c362e3 namespace: kserve-ci-e2e-test resourceVersion: "13856" uid: 86f4c5b5-a331-4821-bcdb-cedb7b088177 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13860" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a716281109ca64 namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: 3da20c95-49f3-4107-a8e9-0d307c258b1a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e10f72195377161ccb6796832df2f49e2c9a07a2917a79cba93118f050c0f8f" metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a716281264f118 namespace: kserve-ci-e2e-test resourceVersion: "13977" uid: 6c71183a-d84f-4b46-b28b-dc35fa48e9ee reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e10f72195377161ccb6796832df2f49e2c9a07a2917a79cba93118f050c0f8f" in 5.324s (5.324s including waiting). Image size: 301288371 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a716294fc51ebf namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: fb92d67e-2214-4a83-8547-5173585181e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a71629542201c7 namespace: kserve-ci-e2e-test resourceVersion: "14048" uid: f0f531b3-3a09-4d0e-8a1c-86806250edeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a7162954bb1573 namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: f3a8d5e9-9834-44a7-9393-6339dd2bb9c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:29Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" metadata: creationTimestamp: "2026-04-17T08:02:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:29Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a7162a768e5f09 namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 9dec897d-2e69-4e8c-9ec8-4b08522199ee reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:39Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" in 9.495s (9.495s including waiting). Image size: 1560613804 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:39Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a7162cac823c7b namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 5d6c710f-8acb-4460-8ea7-9d38c458b508 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:39Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a7162cb1c9bd11 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 03f5fc7c-5474-4be9-8a15-57f0a1aa2432 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:02:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:39Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a7162cb24474ee namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: e856780b-4947-4a86-a49f-3f6e4b575fbe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:04:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:30Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a7162ccdefd927 namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: 8fbb95e8-2478-4bb6-8e15-2e365af9f303 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: dac5afcb-a1b8-43e1-8ffa-f1579954ac75 kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: isvc-sklearn-graph-1-predictor-69d5d56664-444sz.18a716464bbabee4 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 024e5f33-21ba-4d56-9bdf-d222e4efb3bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-69d5d56664 namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: 0107d9ad-44a5-4dd2-be14-886298725a5f kind: Event lastTimestamp: "2026-04-17T08:02:18Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-69d5d56664-444sz' metadata: creationTimestamp: "2026-04-17T08:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:18Z" name: isvc-sklearn-graph-1-predictor-69d5d56664.18a71627f5595958 namespace: kserve-ci-e2e-test resourceVersion: "13853" uid: b1abaeb8-2200-45ee-bc82-d2c8f91a52bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: ccec5940-ea6b-4180-bb7e-c712d91cd51e kind: Event lastTimestamp: "2026-04-17T08:02:18Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-69d5d56664 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:18Z" name: isvc-sklearn-graph-1-predictor.18a71627f1cd0bfa namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 3b1a33a3-d95a-47ee-877f-316df7f0bcbd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:02:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13861" uid: 317bcb1b-216b-4c58-b638-ae097f8f4169 kind: Event lastTimestamp: "2026-04-17T08:03:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:03Z" name: isvc-sklearn-graph-1-predictor.18a7162b76e29b3c namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: a69af6be-608d-429b-9872-eab09de87762 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:02:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13861" uid: 317bcb1b-216b-4c58-b638-ae097f8f4169 kind: Event lastTimestamp: "2026-04-17T08:03:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:03Z" name: isvc-sklearn-graph-1-predictor.18a7162b774cf39b namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: c952b34b-6735-42ee-b1e9-7c9dcc890067 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:03:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 317bcb1b-216b-4c58-b638-ae097f8f4169 kind: Event lastTimestamp: "2026-04-17T08:04:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:04Z" name: isvc-sklearn-graph-1-predictor.18a71635f4189dc8 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: e25764ff-72a2-41a1-9068-f1855281efa1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:03:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 317bcb1b-216b-4c58-b638-ae097f8f4169 kind: Event lastTimestamp: "2026-04-17T08:04:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:04Z" name: isvc-sklearn-graph-1-predictor.18a71635f476cf71 namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: c96fc0c2-0dfc-4c00-bc95-7a7fc384038f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T08:02:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: 50440c6b-0542-4493-ba9c-420801ad21bf kind: Event lastTimestamp: "2026-04-17T08:03:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:50Z" name: isvc-sklearn-graph-1.18a71627f7eb03dc namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 320526cd-c76b-46d9-a95f-fe516735a183 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 50440c6b-0542-4493-ba9c-420801ad21bf kind: Event lastTimestamp: "2026-04-17T08:03:50Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-17T08:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:50Z" name: isvc-sklearn-graph-1.18a7163d59dba05d namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: db15f20d-7c7f-492c-b446-2267433aeb59 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:02:19.373889Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162811c9e511 namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: 56be73f9-3c5b-4ae8-93de-3e6d98ea89a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a716282b16c7c4 namespace: kserve-ci-e2e-test resourceVersion: "13989" uid: e05f0214-6f92-4640-a5ee-cf3d391abee8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e10f72195377161ccb6796832df2f49e2c9a07a2917a79cba93118f050c0f8f" metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a716282c43ef62 namespace: kserve-ci-e2e-test resourceVersion: "13991" uid: a8bd68c1-865c-40e9-89c8-ad5c25230b51 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e10f72195377161ccb6796832df2f49e2c9a07a2917a79cba93118f050c0f8f" in 4.867s (4.867s including waiting). Image size: 301288371 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a716294e6008ba namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: b4ac95c8-e139-4044-875f-4d6a6cdf9921 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a71629540d07fb namespace: kserve-ci-e2e-test resourceVersion: "14047" uid: d1eece17-656f-4a29-9811-772ea47ba2a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162954c3559b namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 58887d42-f57e-46fe-ae70-dc633d4dbde2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" metadata: creationTimestamp: "2026-04-17T08:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:31Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162aee4139d2 namespace: kserve-ci-e2e-test resourceVersion: "14124" uid: f88051ef-9064-42a1-84b5-7b2a0f046b51 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:39Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1374" in 7.503s (7.503s including waiting). Image size: 1560613804 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:39Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162cad7cf9d0 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: eda67d6f-e474-4587-b10c-67a652039609 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:39Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162cb21907e5 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 0376a89f-0093-4452-ab9f-aa779538ee42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:02:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:39Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162cb290ed09 namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 9855d7e3-dc70-4cf0-a394-022831d31cca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T08:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:03:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:40Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7162ccdd26ad7 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: abe63463-7f69-43b9-869e-65a2113114b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: 9641e509-6813-4e2f-97d4-efc806374a09 kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn.18a7164649022051 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 504e9f4a-d6c7-42cb-b014-76217875c9bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7c54695cb6 namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: f46c9c52-fbab-4bde-9038-a9beec09be1a kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7c54695cb6-mc4zn' metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-2-predictor-7c54695cb6.18a71628113c69e8 namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: 88690edb-76b5-414a-bd5e-35867154d97d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 2439c646-9433-4ea4-ae66-2555571c3ea0 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7c54695cb6 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: isvc-sklearn-graph-2-predictor.18a71628096c3533 namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: bba542fe-c12d-4bde-aaed-23613cb47d46 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: c538fd4a-b2a5-444c-b2be-7ec70c31d260 kind: Event lastTimestamp: "2026-04-17T08:03:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:04Z" name: isvc-sklearn-graph-2-predictor.18a7162b88b47bfa namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 1ec681d6-77cb-4a2e-91a0-2d099b240292 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: c538fd4a-b2a5-444c-b2be-7ec70c31d260 kind: Event lastTimestamp: "2026-04-17T08:03:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:04Z" name: isvc-sklearn-graph-2-predictor.18a7162b8942b328 namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 78a217e9-aaea-44bf-9818-205ee2d6fa29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:03:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: c538fd4a-b2a5-444c-b2be-7ec70c31d260 kind: Event lastTimestamp: "2026-04-17T08:04:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:04Z" name: isvc-sklearn-graph-2-predictor.18a71636056566f7 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 035470db-b99b-4a70-a4bc-c4817cd2c23a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:03:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: c538fd4a-b2a5-444c-b2be-7ec70c31d260 kind: Event lastTimestamp: "2026-04-17T08:04:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:04Z" name: isvc-sklearn-graph-2-predictor.18a7163605e34cf7 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 3e661e47-799a-455c-90db-2318ab4e4103 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13929" uid: e697bea7-5e37-4895-a7a5-9f6c5c29124c kind: Event lastTimestamp: "2026-04-17T08:03:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:48Z" name: isvc-sklearn-graph-2.18a7162809a4bc30 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 0327cb1f-1bb2-460b-b4aa-c0a535f71c88 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: e697bea7-5e37-4895-a7a5-9f6c5c29124c kind: Event lastTimestamp: "2026-04-17T08:03:48Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-17T08:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:48Z" name: isvc-sklearn-graph-2.18a7163ce9d5a8d8 namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 9661c017-a2a3-48ae-8825-3a355f49b9ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:02:19.175101Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13924" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-zwcb5 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:02:19Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a7162805f09ee5 namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: 64882a26-7089-48dc-80b4-b5cc183fd8e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:02:19Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a716281fd8fe2a namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: e21ae511-f1a6-4b0d-a6f7-d5847f215304 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e10f72195377161ccb6796832df2f49e2c9a07a2917a79cba93118f050c0f8f" metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:19Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a7162820fe6a2c namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: 9db0f193-e533-4a52-a370-a87639eee297 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e10f72195377161ccb6796832df2f49e2c9a07a2917a79cba93118f050c0f8f" in 5.008s (5.008s including waiting). Image size: 301288371 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a716294b7fe987 namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 43dd7305-df7e-4aed-8778-ee96fb6a2e99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a716294f46b53a namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: bf6d431b-ba2d-4a97-b1c2-7d4b39f43c0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T08:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:24Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a716294fd5bbd0 namespace: kserve-ci-e2e-test resourceVersion: "14044" uid: 222a666e-f07a-41a7-aad7-3047dc9d799e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:31Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-17T08:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a7162aee8142f5 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: c11842be-04db-4e3c-9ead-bf36c567b709 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:54Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.873s (22.873s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:54Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a7163041d832d7 namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 7b22e24d-e1a7-4d0f-9db8-3f79292d252f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:54Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a7163049ca063b namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: e4ae8d4d-c61c-40b2-a6dc-1434f12133f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:02:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:54Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a716304b2a4aff namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 2e75ea9b-2f56-410f-b273-6e9d595b4e13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:02:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:03:45Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:45Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a716304f0cbbfb namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 2b5315f1-1446-4875-9d94-15d57028b566 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5 namespace: kserve-ci-e2e-test resourceVersion: "13928" uid: ed0e16d2-e569-49ed-8a3c-387d722dbe93 kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: isvc-xgboost-graph-predictor-784cb989b8-zwcb5.18a7164650f6cb18 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 771480d7-8c27-4321-86a4-2ce81876f749 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: cc294eda-8be4-49fe-bae3-c12489d70bf1 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-zwcb5' metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a716280592b366 namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: b37e037f-a240-4930-a572-8914a84b9203 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: 9f707ae4-bc22-4160-9137-650f4b954cc6 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: isvc-xgboost-graph-predictor.18a71628000d5a89 namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: 040b8d75-a870-45ec-ab94-5b4adc5a83e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13897" uid: 65bececd-0261-4c21-b439-a5c4b5d8cf03 kind: Event lastTimestamp: "2026-04-17T08:03:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:19Z" name: isvc-xgboost-graph-predictor.18a7162b7e91668b namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 2194d387-bfba-45bd-8dbd-1ff1c147162b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13897" uid: 65bececd-0261-4c21-b439-a5c4b5d8cf03 kind: Event lastTimestamp: "2026-04-17T08:03:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:19Z" name: isvc-xgboost-graph-predictor.18a7162b7ef30c46 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: d1f0bd67-d2b9-4f69-8c5b-9086e21e15b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:03:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 65bececd-0261-4c21-b439-a5c4b5d8cf03 kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:19Z" name: isvc-xgboost-graph-predictor.18a7163979da0f7e namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: e741e605-3e77-4984-b766-098fcc96c544 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:03:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 65bececd-0261-4c21-b439-a5c4b5d8cf03 kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:19Z" name: isvc-xgboost-graph-predictor.18a716397a37c88f namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: d7f826dd-b370-4bf9-ba5e-c92151412056 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: 862d1676-f920-4fe4-a630-7b4631c4bb7c kind: Event lastTimestamp: "2026-04-17T08:03:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:55Z" name: isvc-xgboost-graph.18a71627ffeb7435 namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: 656c1dc5-1e67-4562-bdbc-68acdba4103e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 862d1676-f920-4fe4-a630-7b4631c4bb7c kind: Event lastTimestamp: "2026-04-17T08:03:55Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-17T08:03:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:55Z" name: isvc-xgboost-graph.18a7163e86e9a030 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: ead028d0-dce0-4e53-a782-2a2aa40bb81d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:04:18.854964Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7b7cd8db88-p4crc to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:04:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:04:18Z" name: model-chainer-7b7cd8db88-p4crc.18a71643e36a6878 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 0853d80a-5f5a-48f0-8b3a-74e968ae9659 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:19Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:04:19Z" name: model-chainer-7b7cd8db88-p4crc.18a71643fca2cc24 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 08dcb3c3-00ff-4038-9d6b-6b35e3abdb08 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:19Z" name: model-chainer-7b7cd8db88-p4crc.18a71643fdd1cbcb namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: c487412f-92e1-4b4a-a4db-bc6f0f7c934f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-17T08:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:19Z" name: model-chainer-7b7cd8db88-p4crc.18a71644022da0b0 namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 0982ee17-cff6-4fa0-b7fd-4bbef559dedd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-17T08:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:19Z" name: model-chainer-7b7cd8db88-p4crc.18a7164402f91628 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: d37660b1-b635-492a-bf35-a6fcdc039fda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: "2026-04-17T08:04:28Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-17T08:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:28Z" name: model-chainer-7b7cd8db88-p4crc.18a716463f76a1c6 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: f1fb0164-9d24-436b-9a13-29532173c6e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b7cd8db88-p4crc namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 3c5b1293-930c-4565-b12f-3ee12905cd3e kind: Event lastTimestamp: "2026-04-17T08:04:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:56Z" name: model-chainer-7b7cd8db88-p4crc.18a71646ceac94fe namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: e84a3c69-649b-437d-ad7c-a03bd19fe108 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7b7cd8db88 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 36fa33df-8f0f-4ce3-b95d-c98f71c511d6 kind: Event lastTimestamp: "2026-04-17T08:04:18Z" message: 'Created pod: model-chainer-7b7cd8db88-p4crc' metadata: creationTimestamp: "2026-04-17T08:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:18Z" name: model-chainer-7b7cd8db88.18a71643e3126f83 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: cca7e244-71d0-4fae-88ab-13a90d39a0a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: a6b7d69b-bb74-4989-948b-62741f256d42 kind: Event lastTimestamp: "2026-04-17T08:04:18Z" message: Scaled up replica set model-chainer-7b7cd8db88 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:18Z" name: model-chainer.18a71643e2356197 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 32d9eed7-82aa-47bf-a513-1acb2b46dedc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 2a9596bd-b288-4c81-95c0-cffc967b643b kind: Event lastTimestamp: "2026-04-17T08:04:18Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-17T08:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:18Z" name: model-chainer.18a71643e6fe4fb6 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 90ee638f-088b-4d8d-a252-ecef63e80540 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 2a9596bd-b288-4c81-95c0-cffc967b643b kind: Event lastTimestamp: "2026-04-17T08:04:18Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:18Z" name: model-chainer.18a71643e8ed7d32 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 0f971490-8c3c-426f-bcfe-9cf2f97590f3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 2a9596bd-b288-4c81-95c0-cffc967b643b kind: Event lastTimestamp: "2026-04-17T08:04:18Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:18Z" name: model-chainer.18a71643e8ede37c namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 5b97c067-1c46-4cdb-95d8-e6760735e863 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:15:04.142584Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-375fe-5b588c76f8-lm94t to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:15:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:15:04Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716da218e155b namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 80c60c9a-41d1-4e74-8d73-f2ad734b8d63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-375fe-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:04Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716da2df2b47b namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 6386b183-de45-4351-8255-820913664868 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:05Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:15:05Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716da5e3fad3d namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: f0c13014-ac43-47f2-a1e9-7ac1278f44a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-375fe} kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:05Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716da5f75f8ae namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 16e9f295-178e-4c39-bd00-5a17e4a7bb1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-375fe} kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:05Z" message: 'Created container: sequence-graph-375fe' metadata: creationTimestamp: "2026-04-17T08:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:05Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716da6292a0c3 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: d32e3fae-313d-4d91-8c7b-39d49a58c077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-375fe} kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:05Z" message: Started container sequence-graph-375fe metadata: creationTimestamp: "2026-04-17T08:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:05Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716da630625ff namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b351d0fc-a7ba-4990-b5e9-7dfc47718156 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-375fe} kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Stopping container sequence-graph-375fe metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716dc7ac3591a namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: f43dafa4-1158-4a37-a51c-c82369fadf43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:15:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-375fe} kind: Pod name: sequence-graph-375fe-5b588c76f8-lm94t namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5d64e4e4-969c-441f-932a-9610f8b815d0 kind: Event lastTimestamp: "2026-04-17T08:15:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:41Z" name: sequence-graph-375fe-5b588c76f8-lm94t.18a716dd0158dfd4 namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: 1e923545-9a4e-43a1-81b3-7d74ccd934bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-375fe-5b588c76f8 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 1c84de8e-c4fe-4f2a-afb4-9996b1722622 kind: Event lastTimestamp: "2026-04-17T08:15:04Z" message: 'Created pod: sequence-graph-375fe-5b588c76f8-lm94t' metadata: creationTimestamp: "2026-04-17T08:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:04Z" name: sequence-graph-375fe-5b588c76f8.18a716da2126ad93 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: f659f75e-2c2d-4965-a291-3787ae36eeaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-375fe namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 5a8bb8e5-c573-4ca2-89e8-0a2f712d92b1 kind: Event lastTimestamp: "2026-04-17T08:15:04Z" message: Scaled up replica set sequence-graph-375fe-5b588c76f8 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:04Z" name: sequence-graph-375fe.18a716da20710a6f namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: ffa6b901-f145-4758-aed8-8f08de38eb05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-375fe namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 96d0cd88-81ec-4142-8d25-91247abcfca8 kind: Event lastTimestamp: "2026-04-17T08:15:04Z" message: InferenceGraph [sequence-graph-375fe] is Ready metadata: creationTimestamp: "2026-04-17T08:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:04Z" name: sequence-graph-375fe.18a716da256aaa20 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 5d437fb9-ed91-4844-9610-5d8e81c936c1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:16:24.442652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-60721-6999cb44f9-wjm7f to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a716ecd3cff0c6 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: 057ca230-a30d-47b8-8673-e7b299880067 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:24Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:16:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-60721-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a716ecde140c87 namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: efb28bf4-ae99-4d78-a657-abd8cef27166 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:25Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:16:25Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:16:25Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a716ed10e33861 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 0f6a8b93-4371-4de7-981d-6d0f4f4e54fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-60721} kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:16:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:16:25Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a716ed12298fc4 namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 94c2de8a-ec7e-49b6-aa30-f66167a99a2d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-60721} kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:16:25Z" message: 'Created container: sequence-graph-60721' metadata: creationTimestamp: "2026-04-17T08:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:16:25Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a716ed1509375f namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: 030d72ba-9991-48f1-9b8b-851e32480312 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-60721} kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:16:25Z" message: Started container sequence-graph-60721 metadata: creationTimestamp: "2026-04-17T08:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:16:25Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a716ed1577c507 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: d01c6e81-f67a-4c31-a723-59fbc5f96d5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-60721} kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Stopping container sequence-graph-60721 metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a7176000d5a65b namespace: kserve-ci-e2e-test resourceVersion: "25062" uid: 4c22809d-73b4-4857-b856-5b889a1137ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-60721} kind: Pod name: sequence-graph-60721-6999cb44f9-wjm7f namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: ea0bfedf-ed45-48b2-80eb-2c4800b08d07 kind: Event lastTimestamp: "2026-04-17T08:25:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:06Z" name: sequence-graph-60721-6999cb44f9-wjm7f.18a717609dd7e574 namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 8311a2ab-ce3b-47bf-a72f-8b9babca4a56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-60721-6999cb44f9 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: edc5ca02-e645-4501-9f9d-cbf6946b65fe kind: Event lastTimestamp: "2026-04-17T08:16:24Z" message: 'Created pod: sequence-graph-60721-6999cb44f9-wjm7f' metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721-6999cb44f9.18a716ecd37cb65e namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 28ef5118-6b03-4a18-add9-1e93f256a2c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-60721 namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: ad58b610-063d-4ab8-a9da-6b2ad2a27689 kind: Event lastTimestamp: "2026-04-17T08:16:24Z" message: Scaled up replica set sequence-graph-60721-6999cb44f9 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721.18a716ecd2c3b6bc namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 4219418b-8dcf-413e-80a4-0057aef6cd3c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-60721 namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: b26a70c9-5642-40aa-b696-9cb82b9db999 kind: Event lastTimestamp: "2026-04-17T08:16:24Z" message: InferenceGraph [sequence-graph-60721] is Ready metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721.18a716ecd767d12e namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: 77b38ac6-61a7-4ae7-a6fc-c785ea8628ba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-60721 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: b26a70c9-5642-40aa-b696-9cb82b9db999 kind: Event lastTimestamp: "2026-04-17T08:16:24Z" message: 'Failed to update status for InferenceGraph "sequence-graph-60721": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-60721": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721.18a716ecd97fc119 namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: 7a8e9eda-cbc5-47e1-a5db-fe96d3e301a1 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-60721 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: b26a70c9-5642-40aa-b696-9cb82b9db999 kind: Event lastTimestamp: "2026-04-17T08:16:24Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-60721": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:24Z" name: sequence-graph-60721.18a716ecd9800795 namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: 206da74a-e348-4333-8814-7c41852d178e reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-60721 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: e908e01f-88d7-4b39-9f27-8ea4dadae479 kind: Event lastTimestamp: "2026-04-17T08:16:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:54Z" name: sequence-graph-60721.18a716f0514b0c7d namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: e6b37fb5-61d4-4f1a-a635-5801933f6d06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-60721 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: e908e01f-88d7-4b39-9f27-8ea4dadae479 kind: Event lastTimestamp: "2026-04-17T08:16:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:54Z" name: sequence-graph-60721.18a716f051b51fd9 namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: b5e9739a-7f65-4edf-b726-e9a95abe4d82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:05:39.173662Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c3bdf-57f79946fd-dp8vh to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:05:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:05:39Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a7165696c87531 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: cf0fad73-f0a5-4eac-90c5-997f0e34f2e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:39Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:05:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-c3bdf-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:39Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a71656a19b699b namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 331cec8a-df61-4cfd-a70d-94ed8dd5c1ab reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:05:40Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:05:40Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a71656d377e809 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 5708ac5d-b143-4de7-9fa6-1943604ef7bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3bdf} kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:05:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:40Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a71656d4aa15fa namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 5278fdd8-cbfb-4500-afe5-e5b67fd388e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3bdf} kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:05:40Z" message: 'Created container: sequence-graph-c3bdf' metadata: creationTimestamp: "2026-04-17T08:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:40Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a71656d8329ec6 namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 7545faf7-ec10-4a3d-8d87-a66a341dd1fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3bdf} kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:05:40Z" message: Started container sequence-graph-c3bdf metadata: creationTimestamp: "2026-04-17T08:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:40Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a71656d8a3bac3 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 5a493a11-c086-4973-8a74-070f5bff00f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3bdf} kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:13:53Z" message: Stopping container sequence-graph-c3bdf metadata: creationTimestamp: "2026-04-17T08:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:53Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a716c9ca81d73e namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: d2a0e00f-d93f-4e57-aaba-c04bd8d434a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3bdf} kind: Pod name: sequence-graph-c3bdf-57f79946fd-dp8vh namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d31c24e7-f6c3-43bc-98b0-1e614dff8438 kind: Event lastTimestamp: "2026-04-17T08:14:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:21Z" name: sequence-graph-c3bdf-57f79946fd-dp8vh.18a716ca6af9694e namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: fd467a2d-e448-4096-92ad-b33972c2f325 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c3bdf-57f79946fd namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 5608a3b7-a3be-4870-9e59-f439a551e460 kind: Event lastTimestamp: "2026-04-17T08:05:39Z" message: 'Created pod: sequence-graph-c3bdf-57f79946fd-dp8vh' metadata: creationTimestamp: "2026-04-17T08:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:39Z" name: sequence-graph-c3bdf-57f79946fd.18a71656965c7668 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 2ff02bf6-c9d6-45d6-9969-d365e482f43d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 94e37b6c-b1e1-413f-a220-b154ff70fa76 kind: Event lastTimestamp: "2026-04-17T08:05:39Z" message: Scaled up replica set sequence-graph-c3bdf-57f79946fd from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:39Z" name: sequence-graph-c3bdf.18a7165695a34798 namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: aab64023-8999-460f-820a-21d6fe7e1698 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 38d13a78-bb1d-46d0-8693-37cd76fc2b47 kind: Event lastTimestamp: "2026-04-17T08:05:39Z" message: InferenceGraph [sequence-graph-c3bdf] is Ready metadata: creationTimestamp: "2026-04-17T08:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:05:39Z" name: sequence-graph-c3bdf.18a716569a4b0c64 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: a836e8db-83fb-437e-8407-b1fe386ee93e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 8ef722da-b1bb-491b-b044-eb9a5ba5e161 kind: Event lastTimestamp: "2026-04-17T08:05:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:54Z" name: sequence-graph-c3bdf.18a7165a140edf2c namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 1751a375-8574-4f11-a8ce-8a0605f87da0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 8ef722da-b1bb-491b-b044-eb9a5ba5e161 kind: Event lastTimestamp: "2026-04-17T08:05:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:54Z" name: sequence-graph-c3bdf.18a7165a146ca6d0 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 84683ad4-a972-4a60-aed4-f162900338a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:06:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 8ef722da-b1bb-491b-b044-eb9a5ba5e161 kind: Event lastTimestamp: "2026-04-17T08:06:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:06:09Z" name: sequence-graph-c3bdf.18a7165d93101f3c namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: 1278b296-3e7d-4d18-bee4-e26387b62de9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:06:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 8ef722da-b1bb-491b-b044-eb9a5ba5e161 kind: Event lastTimestamp: "2026-04-17T08:06:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:06:09Z" name: sequence-graph-c3bdf.18a7165d9371b898 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 6d3f1e94-062a-4eec-988d-66f2e8bb2329 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:25:13.577248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-00a8c-7d99f4c66f-pdqll to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:25:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:25:13Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a7176806b10f46 namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 3f01749f-5031-4ff1-ba82-8b2dfd83359f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-00a8c-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:13Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a7176812f9216b namespace: kserve-ci-e2e-test resourceVersion: "25778" uid: 0827d14d-84b5-40bb-aa4d-1897c3bccf69 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:14Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:14Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:25:14Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a717684362dd90 namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 6c3665f9-f582-45e0-a636-24e8b47fe1d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-00a8c} kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:14Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a71768448ac79f namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: b67feeb8-0b94-4339-b359-57babd09e4b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-00a8c} kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:14Z" message: 'Created container: splitter-graph-00a8c' metadata: creationTimestamp: "2026-04-17T08:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:14Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a71768489ef16e namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 0aaebb54-9b39-4ff8-86da-5f019bab64f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-00a8c} kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:14Z" message: Started container splitter-graph-00a8c metadata: creationTimestamp: "2026-04-17T08:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:14Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a7176849137d23 namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: acfbf3a6-384c-4af0-aa07-c2db721190d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-00a8c} kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Stopping container splitter-graph-00a8c metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:23Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a7176a5e74c21e namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 8b1c58a8-bf2f-4244-8d2e-199dea475d93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-00a8c} kind: Pod name: splitter-graph-00a8c-7d99f4c66f-pdqll namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 556f95b4-0d29-4c0c-bec8-15fac50921c7 kind: Event lastTimestamp: "2026-04-17T08:25:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:51Z" name: splitter-graph-00a8c-7d99f4c66f-pdqll.18a7176b06c97f96 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: 1d8002e1-da31-4b7b-add7-ad6e220113c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-00a8c-7d99f4c66f namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: abc2a0ec-b02a-419b-add1-b8ffe4d850ba kind: Event lastTimestamp: "2026-04-17T08:25:13Z" message: 'Created pod: splitter-graph-00a8c-7d99f4c66f-pdqll' metadata: creationTimestamp: "2026-04-17T08:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:13Z" name: splitter-graph-00a8c-7d99f4c66f.18a717680642cc87 namespace: kserve-ci-e2e-test resourceVersion: "25765" uid: c4e2f4a2-7161-4dba-ba31-22f52dffe041 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-00a8c namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: c5d9c8e8-5394-47a4-9c98-9f6eeaed0b02 kind: Event lastTimestamp: "2026-04-17T08:25:13Z" message: Scaled up replica set splitter-graph-00a8c-7d99f4c66f from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:13Z" name: splitter-graph-00a8c.18a717680530e693 namespace: kserve-ci-e2e-test resourceVersion: "25760" uid: 6d4b2d5e-527f-4365-8ed5-6c445b1e68af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-00a8c namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: 1c3a3b2d-ea7f-4885-a394-3a85389b54d0 kind: Event lastTimestamp: "2026-04-17T08:25:13Z" message: InferenceGraph [splitter-graph-00a8c] is Ready metadata: creationTimestamp: "2026-04-17T08:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:13Z" name: splitter-graph-00a8c.18a7176808eefabf namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: d6471440-56f5-4f36-aa5f-0c1ab7817e48 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:26:33.866118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-32e35-6c6f6d485b-vhhcb to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:26:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:26:33Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a7177ab847f8ea namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: d3d49b79-c3ac-4e02-99ee-04215f2d9ef5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:34Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: "2026-04-17T08:26:34Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:26:34Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a7177add54fb01 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 8cd6bf0f-db50-4b8c-bf1f-74bfe5277a0c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32e35} kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: "2026-04-17T08:26:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:26:34Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a7177ade86eb52 namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 633cf12d-2837-4e93-8c84-8426a82a87ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32e35} kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: "2026-04-17T08:26:34Z" message: 'Created container: splitter-graph-32e35' metadata: creationTimestamp: "2026-04-17T08:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:26:34Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a7177ae12db5e9 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: aa8124df-236d-4422-861b-82e847cb3144 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32e35} kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: "2026-04-17T08:26:34Z" message: Started container splitter-graph-32e35 metadata: creationTimestamp: "2026-04-17T08:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:26:34Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a7177ae19aa588 namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 9c49d24a-68df-47d2-a39b-f66e4e68ae12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:34:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32e35} kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: "2026-04-17T08:34:48Z" message: Stopping container splitter-graph-32e35 metadata: creationTimestamp: "2026-04-17T08:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:34:48Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a717ede54df5c9 namespace: kserve-ci-e2e-test resourceVersion: "30098" uid: f5a561af-0816-4585-a63b-64a6f72996ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:34:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32e35} kind: Pod name: splitter-graph-32e35-6c6f6d485b-vhhcb namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 316b0a75-6515-4cdf-966d-b5d52752202d kind: Event lastTimestamp: "2026-04-17T08:35:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:34:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:35:15Z" name: splitter-graph-32e35-6c6f6d485b-vhhcb.18a717ee678eeaf1 namespace: kserve-ci-e2e-test resourceVersion: "30328" uid: c58db53b-5f04-45d8-b1a1-febd648f3751 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-32e35-6c6f6d485b namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 0e734834-773d-42a0-a49e-7e9df6c48eba kind: Event lastTimestamp: "2026-04-17T08:26:33Z" message: 'Created pod: splitter-graph-32e35-6c6f6d485b-vhhcb' metadata: creationTimestamp: "2026-04-17T08:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:26:33Z" name: splitter-graph-32e35-6c6f6d485b.18a7177ab7d9807d namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 53d35065-64d4-46ca-9d7a-0298f9807a8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 45323c77-a379-4b11-a7bc-06a1dfc4835f kind: Event lastTimestamp: "2026-04-17T08:26:33Z" message: Scaled up replica set splitter-graph-32e35-6c6f6d485b from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:26:33Z" name: splitter-graph-32e35.18a7177ab734c589 namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 98ff9019-56e9-4db3-8579-dfa1a9386916 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26653" uid: aa0e7313-e1c9-44c3-ad7a-a9c1bcf829af kind: Event lastTimestamp: "2026-04-17T08:26:33Z" message: InferenceGraph [splitter-graph-32e35] is Ready metadata: creationTimestamp: "2026-04-17T08:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:33Z" name: splitter-graph-32e35.18a7177abac66c57 namespace: kserve-ci-e2e-test resourceVersion: "26655" uid: 802cef3e-96a4-45a1-82d2-c7b612626e86 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: aa0e7313-e1c9-44c3-ad7a-a9c1bcf829af kind: Event lastTimestamp: "2026-04-17T08:26:33Z" message: 'Failed to update status for InferenceGraph "splitter-graph-32e35": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-32e35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:33Z" name: splitter-graph-32e35.18a7177abcf8bf43 namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: 68be0be4-e841-456e-977f-eaaa423dcad1 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: aa0e7313-e1c9-44c3-ad7a-a9c1bcf829af kind: Event lastTimestamp: "2026-04-17T08:26:33Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-32e35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:33Z" name: splitter-graph-32e35.18a7177abcf9056f namespace: kserve-ci-e2e-test resourceVersion: "26659" uid: 64b30925-0ec5-489b-8754-1b78e08746c7 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:26:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 5e428023-cde5-43ce-b433-47b449258677 kind: Event lastTimestamp: "2026-04-17T08:27:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:27:03Z" name: splitter-graph-32e35.18a7177e36088930 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 36a2e523-e5c5-4507-983e-0efce3d50d5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:26:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 5e428023-cde5-43ce-b433-47b449258677 kind: Event lastTimestamp: "2026-04-17T08:27:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:27:03Z" name: splitter-graph-32e35.18a7177e366b877d namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 8a865b94-1c0d-4f81-bc86-538a24edb679 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:24:03.497563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24704" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a71757b59dea8e namespace: kserve-ci-e2e-test resourceVersion: "24716" uid: ab7e6019-e2d1-49d5-a2a8-a15957fd4fb3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24727" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a71757bd3c7a0c namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 4e6acb1b-43c6-4c81-8157-52c0317cb9e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a71757be554057 namespace: kserve-ci-e2e-test resourceVersion: "24745" uid: 437e6646-c8cd-418f-8924-f6a57551b49f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a71757c5e73a77 namespace: kserve-ci-e2e-test resourceVersion: "24754" uid: a9eb82e2-a6ee-4e0e-904d-c62f3b4dfc76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a71757c65dbd8d namespace: kserve-ci-e2e-test resourceVersion: "24755" uid: 1b86bf08-66c0-49fc-9dd4-30494e467a20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:24:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: "2026-04-17T08:25:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:25Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a71757fc94eead namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: 4e4140f1-995b-4041-8358-587c4fd8f3a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v namespace: kserve-ci-e2e-test resourceVersion: "24708" uid: d41082eb-419f-4b18-8bea-0f1939d92dca kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:23Z" name: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v.18a7176a65eb5a3d namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: 4646bec1-75f7-40e3-ad0e-994ffe53c6c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-00a8c-predictor-5bc8467b76 namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 9210f4de-49e8-401b-8939-19c75978e69e kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'Created pod: success-200-isvc-00a8c-predictor-5bc8467b76-n7f4v' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor-5bc8467b76.18a71757b5112ad9 namespace: kserve-ci-e2e-test resourceVersion: "24709" uid: 0537ec3f-df9b-456b-9f8d-60ff553170a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24680" uid: cc1243e5-8ebc-4177-a3e6-a44928ff37e6 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Scaled up replica set success-200-isvc-00a8c-predictor-5bc8467b76 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c-predictor.18a71757b1c0318e namespace: kserve-ci-e2e-test resourceVersion: "24685" uid: 092c49f9-6af2-4a79-a930-3be64e582187 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:24:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24682" uid: 25cbaf01-f4ff-4e04-86c2-be5ce5c27895 kind: Event lastTimestamp: "2026-04-17T08:24:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:33Z" name: success-200-isvc-00a8c-predictor.18a7175b3062fdc1 namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: 92e956c8-d479-440a-b4a9-d426f745880f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:24:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24682" uid: 25cbaf01-f4ff-4e04-86c2-be5ce5c27895 kind: Event lastTimestamp: "2026-04-17T08:24:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:33Z" name: success-200-isvc-00a8c-predictor.18a7175b30bb0995 namespace: kserve-ci-e2e-test resourceVersion: "25000" uid: ac7c81e0-e5e1-4763-9a6a-b0493ffcd776 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:24:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: 25cbaf01-f4ff-4e04-86c2-be5ce5c27895 kind: Event lastTimestamp: "2026-04-17T08:25:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:24:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:18Z" name: success-200-isvc-00a8c-predictor.18a717622e8e5b0a namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: ff613549-410f-46a4-91f9-665df954fad4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:24:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-00a8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: 25cbaf01-f4ff-4e04-86c2-be5ce5c27895 kind: Event lastTimestamp: "2026-04-17T08:25:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:24:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:18Z" name: success-200-isvc-00a8c-predictor.18a717622ef674af namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 4d97946d-a036-4deb-a990-94083662b3e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: e88a67fd-456d-49df-bac2-93db338d22e6 kind: Event lastTimestamp: "2026-04-17T08:24:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:55Z" name: success-200-isvc-00a8c.18a71757b1c7dd26 namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: e61a8eef-a0ab-4e44-b9ed-e6305ef5243d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: e88a67fd-456d-49df-bac2-93db338d22e6 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-00a8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-00a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c.18a71757b260fb94 namespace: kserve-ci-e2e-test resourceVersion: "24689" uid: 482a39f9-641f-433c-b015-0c1d7e74c5d3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: e88a67fd-456d-49df-bac2-93db338d22e6 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-00a8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-00a8c.18a71757b2613fec namespace: kserve-ci-e2e-test resourceVersion: "24691" uid: 7b2e370f-6b63-4acf-bef8-e5c5a7ba65c6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-00a8c namespace: kserve-ci-e2e-test resourceVersion: "25321" uid: e88a67fd-456d-49df-bac2-93db338d22e6 kind: Event lastTimestamp: "2026-04-17T08:24:55Z" message: InferenceService [success-200-isvc-00a8c] is Ready metadata: creationTimestamp: "2026-04-17T08:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:55Z" name: success-200-isvc-00a8c.18a71763e02fdff9 namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: f2464853-38a6-42ee-8626-e9268147536a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:02:19.068710Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13889" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-08159-predictor-d94d646b6-qnkn5 to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:02:19Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a71627ff993aa0 namespace: kserve-ci-e2e-test resourceVersion: "13896" uid: 8f38b4e4-e979-40c1-8b8c-fa9b85920962 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:02:19Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a7162808f82ff6 namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: b8c02403-bab9-459c-bf83-295d73f324cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:19Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a716280a402ffa namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: fa14a264-3783-45a0-94e8-70ddad43954d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:02:32Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" in 13.208s (13.208s including waiting). Image size: 1332223080 bytes.' metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:32Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a7162b1d897dca namespace: kserve-ci-e2e-test resourceVersion: "14139" uid: dd73dd17-c568-462d-867b-7e21dcc8241f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:02:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:32Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a7162b24409b56 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: ef2cb6c7-af0e-4066-aaed-2ad97ec0bca3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:02:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:02:32Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a7162b24b72a14 namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 74cd9d97-4a84-4b60-ae20-8faa35a8e7d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a7162b334196f8 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: bd35a24b-9a06-430a-90d7-b172ad6e57f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08159-predictor-d94d646b6-qnkn5 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: d6e54b0b-98b9-4519-a041-63eaebb24609 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-08159-predictor-d94d646b6-qnkn5.18a7163df0d6dce4 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 7e3cfcef-9342-4ce0-895c-37a250f8ada4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-08159-predictor-d94d646b6 namespace: kserve-ci-e2e-test resourceVersion: "13875" uid: 917a3161-92cb-4604-a138-56492317f909 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: 'Created pod: success-200-isvc-08159-predictor-d94d646b6-qnkn5' metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: success-200-isvc-08159-predictor-d94d646b6.18a71627ff3bc506 namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: 116027f5-ec8f-45f6-8579-d841a877f001 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "13873" uid: 3fd4fb06-95f6-438a-b545-c74500f97f26 kind: Event lastTimestamp: "2026-04-17T08:02:19Z" message: Scaled up replica set success-200-isvc-08159-predictor-d94d646b6 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:02:19Z" name: success-200-isvc-08159-predictor.18a71627fbe29a5f namespace: kserve-ci-e2e-test resourceVersion: "13878" uid: 4ed38a2b-017b-48fd-a8ed-b68bfaf7bd71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "13874" uid: 0c3af7e3-c3e6-4312-a406-ff6eb9e3b2b0 kind: Event lastTimestamp: "2026-04-17T08:03:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:19Z" name: success-200-isvc-08159-predictor.18a7162b7a606439 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 128e4a0a-80fa-474d-8101-ded396349ce3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:02:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "13874" uid: 0c3af7e3-c3e6-4312-a406-ff6eb9e3b2b0 kind: Event lastTimestamp: "2026-04-17T08:03:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:19Z" name: success-200-isvc-08159-predictor.18a7162b7ac68020 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 6fbf8a0a-8baa-4747-94cf-652575cf6b5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:03:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 0c3af7e3-c3e6-4312-a406-ff6eb9e3b2b0 kind: Event lastTimestamp: "2026-04-17T08:03:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:49Z" name: success-200-isvc-08159-predictor.18a7163975e4f4c4 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 1c44a4bc-ecbc-4709-9d4a-234e5a27bb96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:03:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08159-predictor namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 0c3af7e3-c3e6-4312-a406-ff6eb9e3b2b0 kind: Event lastTimestamp: "2026-04-17T08:03:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:49Z" name: success-200-isvc-08159-predictor.18a7163976951b19 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 1eab70d2-0a03-4690-abad-691a084fd45a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:02:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08159 namespace: kserve-ci-e2e-test resourceVersion: "13866" uid: 8a264fe6-3b26-4026-901d-f170b9bdd6e5 kind: Event lastTimestamp: "2026-04-17T08:03:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:02:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:23Z" name: success-200-isvc-08159.18a71627fbc1ae04 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: da7d8da7-7a7e-40ed-92cf-e661acb5b7f5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08159 namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 8a264fe6-3b26-4026-901d-f170b9bdd6e5 kind: Event lastTimestamp: "2026-04-17T08:03:23Z" message: InferenceService [success-200-isvc-08159] is Ready metadata: creationTimestamp: "2026-04-17T08:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:23Z" name: success-200-isvc-08159.18a7163716d08794 namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 5295f45e-95b4-4257-86ff-bc7ee1d2a892 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:25:23.764589Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25922" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-32e35-predictor-855c4946c6-dv4qj to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:25:23Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a7176a65e78ae3 namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 37d0f255-982f-47be-bd25-3ec3e3f9ebaa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:25:23Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a7176a6ec51eb8 namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 1b8e351e-0757-4942-b80f-cf30515c81db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:23Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a7176a6fea6611 namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 05a8c2e2-cd94-410e-91ec-4c33b737c29c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: "2026-04-17T08:25:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:24Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a7176a77f00ac1 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: 31974098-0a68-4a82-9d7e-9f283ab8da5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: "2026-04-17T08:25:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:24Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a7176a78d37213 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: 058ba7ba-abe8-48fc-a3b5-b82d17a63c34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: "2026-04-17T08:26:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:26:05Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a7176aab9edc9d namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: a4824022-e7df-46eb-bd95-8a8ffe429be3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:34:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: 7b7ce7b8-fccb-4f5a-a259-f908d3d26902 kind: Event lastTimestamp: "2026-04-17T08:34:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:34:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:34:48Z" name: success-200-isvc-32e35-predictor-855c4946c6-dv4qj.18a717edeaf751ad namespace: kserve-ci-e2e-test resourceVersion: "30112" uid: 2fe9e749-d827-4f7b-ae89-61db36659d8c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-32e35-predictor-855c4946c6 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 5f82950a-db0e-4f51-9329-55c6fb27a4cd kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: 'Created pod: success-200-isvc-32e35-predictor-855c4946c6-dv4qj' metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:23Z" name: success-200-isvc-32e35-predictor-855c4946c6.18a7176a658abd2d namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: 47544cfc-58c3-4de7-8ace-b60b5775360e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: a1355c39-ddd7-4745-802c-9d5bbc762813 kind: Event lastTimestamp: "2026-04-17T08:25:23Z" message: Scaled up replica set success-200-isvc-32e35-predictor-855c4946c6 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:23Z" name: success-200-isvc-32e35-predictor.18a7176a640e69aa namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 52e9dbbf-70b9-40ff-a6ea-e86ef3075be4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: a4516628-2f41-4cc2-ba3f-1462b2ba4507 kind: Event lastTimestamp: "2026-04-17T08:25:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:38Z" name: success-200-isvc-32e35-predictor.18a7176de294f48f namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: 57683507-de04-458f-9b95-170debbab827 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: a4516628-2f41-4cc2-ba3f-1462b2ba4507 kind: Event lastTimestamp: "2026-04-17T08:25:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:25:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:38Z" name: success-200-isvc-32e35-predictor.18a7176de2f452fd namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: cafd065a-74d0-4bce-9f20-c47af380a67e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:25:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: a4516628-2f41-4cc2-ba3f-1462b2ba4507 kind: Event lastTimestamp: "2026-04-17T08:26:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:26:38Z" name: success-200-isvc-32e35-predictor.18a7177161f19af0 namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 781c9930-c44b-4fc6-9f3a-9159344e1f7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:25:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32e35-predictor namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: a4516628-2f41-4cc2-ba3f-1462b2ba4507 kind: Event lastTimestamp: "2026-04-17T08:26:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:26:38Z" name: success-200-isvc-32e35-predictor.18a71771624f989d namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 88ba5948-d27e-4f7a-9729-1182e21bee46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:25:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32e35 namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: 34eda7c1-718f-4076-8aea-908c98434bc2 kind: Event lastTimestamp: "2026-04-17T08:26:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:16Z" name: success-200-isvc-32e35.18a7176a63d09728 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: c6f4b7c0-e9d7-4db4-b71c-c5e896b05151 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:26:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32e35 namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: 34eda7c1-718f-4076-8aea-908c98434bc2 kind: Event lastTimestamp: "2026-04-17T08:26:15Z" message: InferenceService [success-200-isvc-32e35] is Ready metadata: creationTimestamp: "2026-04-17T08:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:26:15Z" name: success-200-isvc-32e35.18a717768f1c85fd namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: 432a1ce4-b25b-40e2-8443-1a6d3de82f7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:13:54.093455Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-375fe-predictor-5754d77b4d-dxsfl to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716c9d24d2cd7 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 87770122-b00b-4681-95b2-3c17dfab0501 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716c9da66063f namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: 5d5f3c56-bf55-47ed-900f-90277df2856c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716c9db847103 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 91ac6a7f-e089-46ae-9bff-c9279205eaba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716c9e2a72668 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 71ace655-f8ee-44c4-92c7-47b72343a867 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716c9e316cf47 namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: 7fbbf63e-4a02-455d-9ac9-b47752691d4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: "2026-04-17T08:15:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:15Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716ca0068fdd7 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 87706344-7aeb-431d-9c08-6c559c31779b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: ed31a636-b28e-41cc-80a5-c62b82f1c579 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl.18a716dc8197a036 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 0205bdda-714d-4531-95e3-ab39afe6808f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-375fe-predictor-5754d77b4d namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: 2baff580-06fc-43a5-8dc5-92ff10dca275 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'Created pod: success-200-isvc-375fe-predictor-5754d77b4d-dxsfl' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor-5754d77b4d.18a716c9d1e3b8e6 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 64c88f0f-47ff-4025-9cfa-cc98af7ba3a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: a27b8701-a74c-4029-a00f-535cf2bcd338 kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Scaled up replica set success-200-isvc-375fe-predictor-5754d77b4d from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe-predictor.18a716c9d06ec1bb namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: bef99505-ffef-479a-90a5-f35f5f74121e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: f449e658-11f7-4e74-9695-4224ff61c3ad kind: Event lastTimestamp: "2026-04-17T08:14:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:24Z" name: success-200-isvc-375fe-predictor.18a716cd4f1b4af6 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: ecb076c7-267f-445a-af24-8c59f248f3e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: f449e658-11f7-4e74-9695-4224ff61c3ad kind: Event lastTimestamp: "2026-04-17T08:14:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:24Z" name: success-200-isvc-375fe-predictor.18a716cd4f720625 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: f18b15e9-a752-4175-b5e1-99d3a56b6ae8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:14:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: f449e658-11f7-4e74-9695-4224ff61c3ad kind: Event lastTimestamp: "2026-04-17T08:15:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:09Z" name: success-200-isvc-375fe-predictor.18a716d44cdb7f74 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 3f04a595-bce1-4a05-8a45-479b7c653946 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:14:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-375fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: f449e658-11f7-4e74-9695-4224ff61c3ad kind: Event lastTimestamp: "2026-04-17T08:15:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:09Z" name: success-200-isvc-375fe-predictor.18a716d44d40d3fe namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 31f830a8-1d41-44fb-a41b-1be5eba659f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: dccd4b87-eb0e-4ec2-a587-c9507299a64f kind: Event lastTimestamp: "2026-04-17T08:14:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:45Z" name: success-200-isvc-375fe.18a716c9d07aa8dd namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 75756ae3-8171-493a-9161-de8f3579cccf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: dccd4b87-eb0e-4ec2-a587-c9507299a64f kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-375fe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-375fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe.18a716c9d0ebb8c2 namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: d3bcc274-53c8-419a-b42e-a39019fbda8c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: dccd4b87-eb0e-4ec2-a587-c9507299a64f kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-375fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-375fe.18a716c9d0ec7441 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 38a355b8-b42c-4385-b9b5-6ac16dcb8298 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-375fe namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: dccd4b87-eb0e-4ec2-a587-c9507299a64f kind: Event lastTimestamp: "2026-04-17T08:14:45Z" message: InferenceService [success-200-isvc-375fe] is Ready metadata: creationTimestamp: "2026-04-17T08:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:45Z" name: success-200-isvc-375fe.18a716d5e413da22 namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: d81243c6-66de-4c32-8c50-d5e8bd520794 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:13:18.306702Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-390bf-predictor-695484878d-z98cr to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716c17d3eb543 namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: c7f633f6-cab6-47ae-abaf-bcaa06fec846 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716c1850e9597 namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: 4068ea4b-825d-4f59-bbe5-8c6950fbb675 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716c1863c7057 namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: 543479d8-8948-4284-a65e-9250ef2322c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716c18cc031d8 namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 26a09b76-8a7b-436e-8152-af97f2ad1f89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716c18d300f85 namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: 8eb1a427-fd40-4305-92cf-a73aea9dc0a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: "2026-04-17T08:14:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:39Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716c1976c054d namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 603a0bdd-a696-41ba-82a4-54953d3b7bb2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390bf-predictor-695484878d-z98cr namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 80882535-b938-4c10-8df6-f9e0c65e9c9d kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-390bf-predictor-695484878d-z98cr.18a716d42beea9fe namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 3584df16-cbf9-40d1-a960-9bec62ce8c1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-390bf-predictor-695484878d namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: dbc3b94e-1ed4-4749-9453-a107a05b0aca kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'Created pod: success-200-isvc-390bf-predictor-695484878d-z98cr' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor-695484878d.18a716c17cd7402b namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: cf44d258-ddca-4805-b24c-2de46798c357 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-390bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: d47872fb-94a1-4199-8ec0-ae591f2cd60e kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Scaled up replica set success-200-isvc-390bf-predictor-695484878d from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf-predictor.18a716c1794701a9 namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: 68324972-54c5-4d5a-9a20-d650b9f2deef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 966feb7e-e9f6-4e54-9db1-66660176cf07 kind: Event lastTimestamp: "2026-04-17T08:14:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:33Z" name: success-200-isvc-390bf-predictor.18a716c4f7c2613f namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 2cd6ac36-c2d8-4f56-a051-cd51c1b003c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:13:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390bf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 966feb7e-e9f6-4e54-9db1-66660176cf07 kind: Event lastTimestamp: "2026-04-17T08:14:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:33Z" name: success-200-isvc-390bf-predictor.18a716c4f823c94f namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: cb6485b1-e89d-48d0-915c-f1f25e915947 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 8f25aa11-f5c8-42ec-8977-1eaea48e521a kind: Event lastTimestamp: "2026-04-17T08:14:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:09Z" name: success-200-isvc-390bf.18a716c179237265 namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 315e9408-b9cf-4d7f-8c20-3a078d786bf0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 8f25aa11-f5c8-42ec-8977-1eaea48e521a kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-390bf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-390bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf.18a716c1798beba7 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: b0a55b85-7165-4c6b-96c4-be7b891b58b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: 8f25aa11-f5c8-42ec-8977-1eaea48e521a kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-390bf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-390bf.18a716c1798c3de2 namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: b9b781fc-60b5-4cba-b89c-8861b0aa9589 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390bf namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 8f25aa11-f5c8-42ec-8977-1eaea48e521a kind: Event lastTimestamp: "2026-04-17T08:14:09Z" message: InferenceService [success-200-isvc-390bf] is Ready metadata: creationTimestamp: "2026-04-17T08:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:14:09Z" name: success-200-isvc-390bf.18a716cd7afac2c5 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 633de7ff-40f1-42b8-8fa3-1c09b5c22d68 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:03:53.351549Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a7163df34b31b3 namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 6578809b-39e9-4b5c-9c94-9eed21ee2339 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a7163dfbb415f5 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: f1def7e5-6ebf-4358-8589-7fea361bbc0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a7163dfd115190 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 384dcbcc-bcb4-4869-a293-89d96735de2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a7163e00924cb6 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: a27e83f6-0f9c-488a-87f0-b6ac3364edee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a7163e01030d2b namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 8dcbce8f-b049-46f4-86a1-9480fbcd204d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: "2026-04-17T08:04:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:35Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a7163e1d28fc0f namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: a5f4d33d-49fd-43d9-988d-34b39309de10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 74f58849-9030-4931-9db3-1b2cd2a0088b kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v.18a716c17974e7b1 namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: c2842d9f-b73b-4011-8ad8-466f4e9e7cfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4828d-predictor-6ddfbf8d5b namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 648da776-af4e-434a-931c-1a26e294142d kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Created pod: success-200-isvc-4828d-predictor-6ddfbf8d5b-fmz9v' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor-6ddfbf8d5b.18a7163df2c4b1e6 namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 3aa2533e-08cd-4489-a8e5-779b81b765b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 793e431a-b7d4-4bb3-bcc9-43998942f124 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Scaled up replica set success-200-isvc-4828d-predictor-6ddfbf8d5b from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d-predictor.18a7163df015a345 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 05bd349c-1f6e-4762-9a56-a44a93fbd1ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 84ecaca2-97d4-42fc-8abf-0f40b4c239fc kind: Event lastTimestamp: "2026-04-17T08:04:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:08Z" name: success-200-isvc-4828d-predictor.18a716416ecc5dfe namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: e531c9a2-1d3b-45a6-9103-a441764ab3c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 84ecaca2-97d4-42fc-8abf-0f40b4c239fc kind: Event lastTimestamp: "2026-04-17T08:04:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:08Z" name: success-200-isvc-4828d-predictor.18a716416f2ae624 namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: d37439b0-8814-42c1-a048-04de37246c54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 84ecaca2-97d4-42fc-8abf-0f40b4c239fc kind: Event lastTimestamp: "2026-04-17T08:05:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:08Z" name: success-200-isvc-4828d-predictor.18a71644edc6d276 namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 3b9ec944-c5b1-4a2f-8df4-a4d196f109f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4828d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 84ecaca2-97d4-42fc-8abf-0f40b4c239fc kind: Event lastTimestamp: "2026-04-17T08:05:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:08Z" name: success-200-isvc-4828d-predictor.18a71644ee2c5121 namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 27c5ee99-f4b5-4684-93a7-76ba57f64c3d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: d73219f4-f10c-4d86-aa46-01566506cf93 kind: Event lastTimestamp: "2026-04-17T08:04:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:45Z" name: success-200-isvc-4828d.18a7163df02f1827 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: b758d4ac-e854-4301-9dcc-10cbeaee38cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: d73219f4-f10c-4d86-aa46-01566506cf93 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'Failed to update status for InferenceService "success-200-isvc-4828d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4828d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d.18a7163df0b937c3 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: ac9ec8c9-6a11-403c-9c31-e0bac82f237c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: d73219f4-f10c-4d86-aa46-01566506cf93 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4828d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:53Z" name: success-200-isvc-4828d.18a7163df0b9ce27 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: f31d7817-eebf-46b8-a74e-abdd9b2a2860 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4828d namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: d73219f4-f10c-4d86-aa46-01566506cf93 kind: Event lastTimestamp: "2026-04-17T08:04:45Z" message: InferenceService [success-200-isvc-4828d] is Ready metadata: creationTimestamp: "2026-04-17T08:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:04:45Z" name: success-200-isvc-4828d.18a7164a008c1c77 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 85761ead-0268-4b7f-b8c5-2924798a04cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:24:39.309969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-54cf7-predictor-56f798986b-lsftf to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a717600c33e22c namespace: kserve-ci-e2e-test resourceVersion: "25111" uid: 48a674a4-9212-4e53-882c-a57c846e3477 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a7176014a13555 namespace: kserve-ci-e2e-test resourceVersion: "25147" uid: c7e4d640-6018-4e34-a403-f95c52d83896 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a71760160962f3 namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: 5f535c14-e168-4177-991e-2223e9c0e9a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a717601b82e150 namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 2ed8e793-e999-459c-9e26-68cff77e1c8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a717601bf7e73e namespace: kserve-ci-e2e-test resourceVersion: "25157" uid: e06f79c0-204a-483d-9cdd-fac1bf35b7fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: "2026-04-17T08:42:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:42:10Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a7176029048a56 namespace: kserve-ci-e2e-test resourceVersion: "33229" uid: 4067c121-0441-4b22-810e-8e0150ccfd5b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54cf7-predictor-56f798986b-lsftf namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: bf2bd1ce-a247-4c04-bdfc-4e20dbdcb812 kind: Event lastTimestamp: "2026-04-17T08:42:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:42:08Z" name: success-200-isvc-54cf7-predictor-56f798986b-lsftf.18a7185468200a19 namespace: kserve-ci-e2e-test resourceVersion: "33171" uid: ba63644d-3a85-46c9-bcba-247edd361cf2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-54cf7-predictor-56f798986b namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: 25714e52-d43c-49a0-a7e6-26c1a9f5e45d kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'Created pod: success-200-isvc-54cf7-predictor-56f798986b-lsftf' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor-56f798986b.18a717600bcbaec7 namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: 612d280b-c2a9-46ec-98a4-5dab6dd43cbb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: 786d4f49-c751-4929-8d6f-42d6d04f1194 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Scaled up replica set success-200-isvc-54cf7-predictor-56f798986b from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7-predictor.18a717600a2f1a2f namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 4aff1a1e-a29a-4d88-81cd-c3cb09da0b7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: b472a8d2-508c-444b-aa29-f665302db89b kind: Event lastTimestamp: "2026-04-17T08:24:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:54Z" name: success-200-isvc-54cf7-predictor.18a71763889f8739 namespace: kserve-ci-e2e-test resourceVersion: "25292" uid: 88b93248-23e9-485d-82e7-23d435caddca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: b472a8d2-508c-444b-aa29-f665302db89b kind: Event lastTimestamp: "2026-04-17T08:24:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:24:54Z" name: success-200-isvc-54cf7-predictor.18a7176389005f46 namespace: kserve-ci-e2e-test resourceVersion: "25294" uid: 7d0be3da-dd63-4a19-a7b7-31f7f9905ec0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:25:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: b472a8d2-508c-444b-aa29-f665302db89b kind: Event lastTimestamp: "2026-04-17T08:25:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:54Z" name: success-200-isvc-54cf7-predictor.18a7176707a605ac namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 72c0a76b-2e29-46d7-bd64-6d258210325d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:25:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54cf7-predictor namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: b472a8d2-508c-444b-aa29-f665302db89b kind: Event lastTimestamp: "2026-04-17T08:25:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:54Z" name: success-200-isvc-54cf7-predictor.18a7176708032918 namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 26bb1fb6-73a7-4698-9a1d-87fd8a6898f7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: b23ddf03-4ffd-4eea-bfdc-ed240b37dcd8 kind: Event lastTimestamp: "2026-04-17T08:25:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:30Z" name: success-200-isvc-54cf7.18a717600a190d06 namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 445f7772-af61-4570-8a72-cbfa55cf484c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: b23ddf03-4ffd-4eea-bfdc-ed240b37dcd8 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'Failed to update status for InferenceService "success-200-isvc-54cf7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-54cf7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7.18a717600aae90e0 namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 803929cd-3828-487a-9881-5faf75bf1e22 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: b23ddf03-4ffd-4eea-bfdc-ed240b37dcd8 kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-54cf7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-54cf7.18a717600aaef108 namespace: kserve-ci-e2e-test resourceVersion: "25104" uid: f9211d06-a48d-45c7-b805-3beef519632b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "26061" uid: b23ddf03-4ffd-4eea-bfdc-ed240b37dcd8 kind: Event lastTimestamp: "2026-04-17T08:25:30Z" message: InferenceService [success-200-isvc-54cf7] is Ready metadata: creationTimestamp: "2026-04-17T08:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:30Z" name: success-200-isvc-54cf7.18a7176c0c632524 namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: bcec761c-2ce2-48df-990d-7e42a93bf7db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:15:14.349629Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-60721-predictor-56cc48fd48-qqh8q to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a716dc81f13c29 namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 885d1778-ea97-4d31-b3c8-cf4f94d1b3fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a716dc896333a9 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: e69f5a80-2dee-477c-8956-b278dcc8d847 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a716dc8ac97b5f namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: a5a22600-58c3-4065-b071-28613451ea41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a716dc8eddfe13 namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: be7d0383-a884-400d-9bd7-9ee46c8980df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a716dc8f53362a namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 68fda48d-d962-4f3c-bef4-1827f63256d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:15:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: "2026-04-17T08:15:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:56Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a716dcafa3bdc1 namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 4b852cce-da4d-402b-b448-36f658209d82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 7f094e49-fecd-4e33-8e15-a761ff85682c kind: Event lastTimestamp: "2026-04-17T08:24:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:39Z" name: success-200-isvc-60721-predictor-56cc48fd48-qqh8q.18a717600a9eb7f1 namespace: kserve-ci-e2e-test resourceVersion: "25102" uid: cf573688-2d4c-474c-8ceb-5e676d6b0e4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-60721-predictor-56cc48fd48 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 65006c3c-7868-497b-bcd4-1f70596e7ae0 kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: 'Created pod: success-200-isvc-60721-predictor-56cc48fd48-qqh8q' metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor-56cc48fd48.18a716dc819b2a85 namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: 2d3d0b40-1d96-4086-b0fd-f1f728d0a07e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: c66058b5-9a19-4c71-931e-b6d287baafab kind: Event lastTimestamp: "2026-04-17T08:15:14Z" message: Scaled up replica set success-200-isvc-60721-predictor-56cc48fd48 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:14Z" name: success-200-isvc-60721-predictor.18a716dc7fd93302 namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 3df72e17-5f9c-4d5b-a7fc-e53efebc20a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 4290510e-fa0e-4bbc-94ed-656a089c30c8 kind: Event lastTimestamp: "2026-04-17T08:15:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:29Z" name: success-200-isvc-60721-predictor.18a716dffe4a2fa4 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: c090e4a3-861f-4f03-8a52-ba9fee17bf5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 4290510e-fa0e-4bbc-94ed-656a089c30c8 kind: Event lastTimestamp: "2026-04-17T08:15:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:29Z" name: success-200-isvc-60721-predictor.18a716dffebc194c namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: a8f11d6e-ca57-4e79-a0c3-767392e823e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:15:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 4290510e-fa0e-4bbc-94ed-656a089c30c8 kind: Event lastTimestamp: "2026-04-17T08:16:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:29Z" name: success-200-isvc-60721-predictor.18a716e37dec8e16 namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: 80be4e95-ad1d-4792-bcf0-5df783fabc8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:15:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60721-predictor namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 4290510e-fa0e-4bbc-94ed-656a089c30c8 kind: Event lastTimestamp: "2026-04-17T08:16:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:16:29Z" name: success-200-isvc-60721-predictor.18a716e37e4da3ed namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: a0e6a957-3385-4919-80f5-f313d116ee82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T08:15:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60721 namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: c777b509-9c1c-4591-8508-694f8eb42cbc kind: Event lastTimestamp: "2026-04-17T08:16:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:06Z" name: success-200-isvc-60721.18a716dc7fb55084 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 1cbdfd52-fce7-4207-b811-36c0c6d34f9d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60721 namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: c777b509-9c1c-4591-8508-694f8eb42cbc kind: Event lastTimestamp: "2026-04-17T08:16:06Z" message: InferenceService [success-200-isvc-60721] is Ready metadata: creationTimestamp: "2026-04-17T08:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:16:06Z" name: success-200-isvc-60721.18a716e8933fe6fc namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 83938d0c-e94b-4cba-9344-8954c2ae6e66 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:14:38.559679Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-65d2e-predictor-56f76468b8-qqxkk to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a716d42cb1eb2e namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: c4bd3f7f-2118-4023-8854-47b5a6b436ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a716d434ffedb4 namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 530772e1-8b38-45bf-8d6c-0b58b6115a8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a716d43629d4a3 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 406ef117-1c42-4f0f-9cbf-5c4ae8497b37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a716d43a1faab9 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: f944a57b-8b79-4564-8d7e-9715f7a9d35f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a716d43aa744be namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 26027ec0-16c1-4e00-929f-8f37e5f77857 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:14:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: "2026-04-17T08:15:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:15:20Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a716d4479a9899 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: c5bd2ad2-a6be-4d2f-b8a7-8fae51d51a6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:24:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 9db16dfd-98b0-46d1-a6a1-779cec048066 kind: Event lastTimestamp: "2026-04-17T08:24:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:24:03Z" name: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk.18a71757b1cb01aa namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: e56da5a6-1cbc-4c4e-b6ad-65fe68d73f68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-65d2e-predictor-56f76468b8 namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 43557305-702b-4096-beef-be988555330c kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: 'Created pod: success-200-isvc-65d2e-predictor-56f76468b8-qqxkk' metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor-56f76468b8.18a716d42c307507 namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: d3002729-953b-42d5-acd5-dcc166824527 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: f459295e-fad9-49af-8246-ce4b4012bf90 kind: Event lastTimestamp: "2026-04-17T08:14:38Z" message: Scaled up replica set success-200-isvc-65d2e-predictor-56f76468b8 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:14:38Z" name: success-200-isvc-65d2e-predictor.18a716d42aae180e namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: 233dac09-a8b5-450d-a302-aaf0224ef359 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:14:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 57fa0431-ef63-43a8-abf8-8887cef6b250 kind: Event lastTimestamp: "2026-04-17T08:15:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:08Z" name: success-200-isvc-65d2e-predictor.18a716d7a9bcf83c namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 4e71b04d-7cb6-410e-aa1e-4f65f6a8135c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:14:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 57fa0431-ef63-43a8-abf8-8887cef6b250 kind: Event lastTimestamp: "2026-04-17T08:15:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:14:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:08Z" name: success-200-isvc-65d2e-predictor.18a716d7aa19b5e8 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 3d9d6a4e-a55b-4246-8fb0-491f40e13c5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:15:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 57fa0431-ef63-43a8-abf8-8887cef6b250 kind: Event lastTimestamp: "2026-04-17T08:15:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:38Z" name: success-200-isvc-65d2e-predictor.18a716dea78824d3 namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: 4c9cc994-e873-46e1-b776-b4c08a9c2f85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:15:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65d2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 57fa0431-ef63-43a8-abf8-8887cef6b250 kind: Event lastTimestamp: "2026-04-17T08:15:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:15:38Z" name: success-200-isvc-65d2e-predictor.18a716dea7eaac92 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: f4041dcb-7c81-44b7-b674-9eaffc069e04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:14:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65d2e namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: bbed96fb-15a3-4320-a4a0-eb94b3b4b22d kind: Event lastTimestamp: "2026-04-17T08:15:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:30Z" name: success-200-isvc-65d2e.18a716d42ab2cb49 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: d1165721-6d71-4f1a-a72a-ec2a3dcbeadd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65d2e namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: bbed96fb-15a3-4320-a4a0-eb94b3b4b22d kind: Event lastTimestamp: "2026-04-17T08:15:30Z" message: InferenceService [success-200-isvc-65d2e] is Ready metadata: creationTimestamp: "2026-04-17T08:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:15:30Z" name: success-200-isvc-65d2e.18a716e02b37630a namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 7af6e5a0-03ed-4d8e-a19b-0025a8a7a21a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:04:29.172568Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf to ip-10-0-137-165.ec2.internal metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a716464a648456 namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: 971ece74-9bce-4988-9aac-b710dee2d973 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a7164652fb79cc namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 98363805-91fe-41ce-aa79-47745c50a22b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1374" already present on machine metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a716465431ff68 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: a25141cc-525c-439a-9e28-4f629f3b1126 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a716465bd88206 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 01a7e320-5669-48a3-9f79-a2e3a5c78fca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a716465c4a7dc5 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 7f917f58-6d63-4219-99af-f5cce9a6f4bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: "2026-04-17T08:05:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T08:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:11Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a7164685cbb238 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 10f7478a-f6b9-4fdd-8f18-039c8129836e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: eaf732ca-639c-4439-94bd-0c696e6a014e kind: Event lastTimestamp: "2026-04-17T08:13:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T08:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:54Z" name: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf.18a716c9d08646bd namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 3b1e1991-3b68-4f82-a2ba-c72815b7dc1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-165.ec2.internal source: component: kubelet host: ip-10-0-137-165.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3bdf-predictor-6fc8445484 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 5e5dd23b-4302-4eb7-b488-ed8536694099 kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: 'Created pod: success-200-isvc-c3bdf-predictor-6fc8445484-6hzdf' metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor-6fc8445484.18a7164649ff8488 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: c844d48d-8334-42b2-a973-338a239cf968 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 7b789af7-7445-4b67-96a0-e89371f49cbd kind: Event lastTimestamp: "2026-04-17T08:04:29Z" message: Scaled up replica set success-200-isvc-c3bdf-predictor-6fc8445484 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:04:29Z" name: success-200-isvc-c3bdf-predictor.18a716464888a353 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 5f69145e-aed3-4b70-8e66-3fd7bb71b97e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 53634021-5ad4-4b23-881a-41afcf15b48a kind: Event lastTimestamp: "2026-04-17T08:05:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:14Z" name: success-200-isvc-c3bdf-predictor.18a71649c741c27a namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 798a80e9-c950-48d8-8d73-3aac8ab77866 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:04:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 53634021-5ad4-4b23-881a-41afcf15b48a kind: Event lastTimestamp: "2026-04-17T08:05:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:14Z" name: success-200-isvc-c3bdf-predictor.18a71649c79ef573 namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 8a70804c-767d-4b4a-bb5a-c4f272fcd6f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: 53634021-5ad4-4b23-881a-41afcf15b48a kind: Event lastTimestamp: "2026-04-17T08:05:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:59Z" name: success-200-isvc-c3bdf-predictor.18a71654443a7376 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 02933a90-31aa-4c6b-837e-d426497a99b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T08:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3bdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: 53634021-5ad4-4b23-881a-41afcf15b48a kind: Event lastTimestamp: "2026-04-17T08:05:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:59Z" name: success-200-isvc-c3bdf-predictor.18a71654449cec60 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: df2b4e94-9f66-41a4-b099-b083729cd718 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:04:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 6aeab2d6-af7c-4a8d-bc9e-906847e0de12 kind: Event lastTimestamp: "2026-04-17T08:05:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T08:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:05:21Z" name: success-200-isvc-c3bdf.18a71646485f4b2c namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 3252bd27-dbce-4048-931a-1c94e2327f19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3bdf namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 6aeab2d6-af7c-4a8d-bc9e-906847e0de12 kind: Event lastTimestamp: "2026-04-17T08:05:21Z" message: InferenceService [success-200-isvc-c3bdf] is Ready metadata: creationTimestamp: "2026-04-17T08:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:05:21Z" name: success-200-isvc-c3bdf.18a7165269798624 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: a559088c-a517-4fb1-89a9-f34602c0bb62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:03:38.912508Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-08159-8656fb4779-lqhtj to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:03:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:03:38Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163a96a8e37a namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: f0ba9869-bb72-491b-8d7b-3b3185c82d42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-08159-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:39Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163aa1dc01dd namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 1b496a69-962f-4d21-8ffe-d94e1887eead reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:39Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:03:39Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163ad33ea6a3 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: cff787d8-8511-4281-b839-c3865f7c81e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-08159} kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:39Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" metadata: creationTimestamp: "2026-04-17T08:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:39Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163ad46b8dcd namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: a5a6f401-497e-422f-945b-cce6badc1cae reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-08159} kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" in 2.293s (2.293s including waiting). Image size: 216233559 bytes.' metadata: creationTimestamp: "2026-04-17T08:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:42Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163b5d2244de namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 7e5cdb8e-dd9c-48a5-a28c-427bc1833f66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-08159} kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:42Z" message: 'Created container: switch-graph-08159' metadata: creationTimestamp: "2026-04-17T08:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:42Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163b6016c0c3 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: d8ca86cb-f391-4563-b37d-0b2f518bf210 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-08159} kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:42Z" message: Started container switch-graph-08159 metadata: creationTimestamp: "2026-04-17T08:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:42Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163b60865ab1 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 070387c3-5220-491c-874c-7d685148d372 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-08159} kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:03:53Z" message: Stopping container switch-graph-08159 metadata: creationTimestamp: "2026-04-17T08:03:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:03:53Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163de9dc3111 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: cf8c1f21-ed1e-449d-b967-870d21b5de4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-08159} kind: Pod name: switch-graph-08159-8656fb4779-lqhtj namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: fcee06dc-f1a7-46af-a694-5b4320c29862 kind: Event lastTimestamp: "2026-04-17T08:04:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:04:19Z" name: switch-graph-08159-8656fb4779-lqhtj.18a7163e29f4c450 namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: d935d7c5-ed04-4aa1-8017-689fe21f7e8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-08159-8656fb4779 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 17ee4d45-0b71-47b2-852d-6c73d1df16ba kind: Event lastTimestamp: "2026-04-17T08:03:38Z" message: 'Created pod: switch-graph-08159-8656fb4779-lqhtj' metadata: creationTimestamp: "2026-04-17T08:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:38Z" name: switch-graph-08159-8656fb4779.18a7163a963c9d85 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: c36ebbc5-74ff-41d7-9e5a-c95c2488463d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-08159 namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: d2a135c8-1ee6-4376-8326-a7dc5f7c31d9 kind: Event lastTimestamp: "2026-04-17T08:03:38Z" message: Scaled up replica set switch-graph-08159-8656fb4779 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:03:38Z" name: switch-graph-08159.18a7163a958aa3bd namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 3cbb7179-2c06-485c-8aef-4baa808ef0ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:03:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-08159 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: ac6758bf-7e99-4ac5-b628-1bf8aa89ab58 kind: Event lastTimestamp: "2026-04-17T08:03:38Z" message: InferenceGraph [switch-graph-08159] is Ready metadata: creationTimestamp: "2026-04-17T08:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:03:38Z" name: switch-graph-08159.18a7163a98db8b3b namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: 059f96af-a260-45b8-9914-55eaace2146e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:05:03.430635Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4828d-85cdb4b486-9xjzw to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:05:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:05:03Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a7164e4455261c namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 76b382c5-f7b2-43d4-9510-93de15ced947 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:03Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:05:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4828d-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:03Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a7164e4da630b5 namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: e3ebb63f-a247-42d7-8561-bf7cdde47427 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:05:04Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:05:04Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a7164e8104e4dd namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: a91fc599-7c1d-49e8-9522-07b554b9746a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4828d} kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:05:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:04Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a7164e824ad9a3 namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: c68b57ee-e647-45bd-a16f-93ce50320527 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4828d} kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:05:04Z" message: 'Created container: switch-graph-4828d' metadata: creationTimestamp: "2026-04-17T08:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:04Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a7164e8535d4a4 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 2cb23cdd-de5a-44df-8e19-c0044f93188a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4828d} kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:05:04Z" message: Started container switch-graph-4828d metadata: creationTimestamp: "2026-04-17T08:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:05:04Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a7164e85a82dff namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: ef759e76-0d24-4eda-979f-e4099032f1d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:13:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4828d} kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:13:18Z" message: Stopping container switch-graph-4828d metadata: creationTimestamp: "2026-04-17T08:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:18Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a716c173954730 namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: 80ccc7e2-ea43-433f-bc7c-823377073ead reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T08:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4828d} kind: Pod name: switch-graph-4828d-85cdb4b486-9xjzw namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: bdaad542-c3c4-449a-b6bb-7e2448ba506a kind: Event lastTimestamp: "2026-04-17T08:13:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:13:45Z" name: switch-graph-4828d-85cdb4b486-9xjzw.18a716c202477269 namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 96ce4ced-04be-41a2-a368-7a6891026008 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4828d-85cdb4b486 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: fce2108a-adc7-4863-9a2f-78812641c775 kind: Event lastTimestamp: "2026-04-17T08:05:03Z" message: 'Created pod: switch-graph-4828d-85cdb4b486-9xjzw' metadata: creationTimestamp: "2026-04-17T08:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:03Z" name: switch-graph-4828d-85cdb4b486.18a7164e43f28a07 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 12c1d715-7918-42cf-83ec-9121c654aa0e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4828d namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b5ac409e-b2ad-4a58-961a-648175784226 kind: Event lastTimestamp: "2026-04-17T08:05:03Z" message: Scaled up replica set switch-graph-4828d-85cdb4b486 from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:03Z" name: switch-graph-4828d.18a7164e430ff8c3 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 274520e0-94c0-4999-aebb-4a68142f6579 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4828d namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 6ed6ef75-2267-41da-8c06-234af2cb803a kind: Event lastTimestamp: "2026-04-17T08:05:03Z" message: InferenceGraph [switch-graph-4828d] is Ready metadata: creationTimestamp: "2026-04-17T08:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:05:03Z" name: switch-graph-4828d.18a7164e47ce04bf namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: afb60e90-c817-4576-9e85-bad3822964d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4828d namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 757e0388-7bb8-4681-92ab-e059003cfbe8 kind: Event lastTimestamp: "2026-04-17T08:05:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:05:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:18Z" name: switch-graph-4828d.18a71651c183c413 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 7bd68716-f557-44cb-8d15-256f6da51161 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:05:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4828d namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 757e0388-7bb8-4681-92ab-e059003cfbe8 kind: Event lastTimestamp: "2026-04-17T08:05:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T08:05:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:05:18Z" name: switch-graph-4828d.18a71651c1b8480e namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 421d7502-7284-42f7-98de-0f7ce8c6390e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T08:25:49.401267Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-54cf7-669b44b5bb-25b6l to ip-10-0-141-224.ec2.internal metadata: creationTimestamp: "2026-04-17T08:25:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T08:25:49Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a717705df832ea namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: b5c3d98c-b9d8-40d9-af04-a46490ad8cfd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8587476c78-qc8t9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:25:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-54cf7-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T08:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:49Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a7177066bbfbcf namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 98169651-abfe-4d51-8a7f-0a892959f55c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:25:50Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T08:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T08:25:50Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a717709b143b7e namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: 680e5b3b-128b-4d78-8602-8f16dc515db5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54cf7} kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:25:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:01e079bf4e7c8c9fa0b9aff85868fd044b4e84650f0ee098893627437ef45803" already present on machine metadata: creationTimestamp: "2026-04-17T08:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:50Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a717709c424ea6 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 253a4fc2-aba0-462c-ac63-6b55699e331c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54cf7} kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:25:50Z" message: 'Created container: switch-graph-54cf7' metadata: creationTimestamp: "2026-04-17T08:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:50Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a717709f350999 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 48639f89-08e9-4252-a6f8-c0e82b7ab06d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54cf7} kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:25:50Z" message: Started container switch-graph-54cf7 metadata: creationTimestamp: "2026-04-17T08:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:25:50Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a717709fa64ea4 namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 123dc046-a803-424b-bb11-c1dbe6d3c83b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54cf7} kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:42:08Z" message: Stopping container switch-graph-54cf7 metadata: creationTimestamp: "2026-04-17T08:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:42:08Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a7185460bbbbbb namespace: kserve-ci-e2e-test resourceVersion: "33158" uid: ba4f52e7-978b-40a6-970b-4dfc24fdacaa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T08:42:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54cf7} kind: Pod name: switch-graph-54cf7-669b44b5bb-25b6l namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 027375f1-69ee-41d8-9a78-9c7da61f7450 kind: Event lastTimestamp: "2026-04-17T08:42:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T08:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T08:42:26Z" name: switch-graph-54cf7-669b44b5bb-25b6l.18a718550c2e1f81 namespace: kserve-ci-e2e-test resourceVersion: "33358" uid: 9afd49c9-3164-40e6-856a-9b1a2b963572 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-224.ec2.internal source: component: kubelet host: ip-10-0-141-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-54cf7-669b44b5bb namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: bdecd17a-0463-4e68-aeb1-af21723b8ca6 kind: Event lastTimestamp: "2026-04-17T08:25:49Z" message: 'Created pod: switch-graph-54cf7-669b44b5bb-25b6l' metadata: creationTimestamp: "2026-04-17T08:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:49Z" name: switch-graph-54cf7-669b44b5bb.18a717705d8faa7f namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: af37bd6e-1769-4d09-b4e0-4eeea616cfd3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: edc3df61-2de0-45d2-8338-ae2bea8be534 kind: Event lastTimestamp: "2026-04-17T08:25:49Z" message: Scaled up replica set switch-graph-54cf7-669b44b5bb from 0 to 1 metadata: creationTimestamp: "2026-04-17T08:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:25:49Z" name: switch-graph-54cf7.18a717705ce3f5db namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: c84a7fde-1117-4449-a425-19e919bda933 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T08:25:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 0b224355-2d5c-4700-a5c4-86afe5c2e0df kind: Event lastTimestamp: "2026-04-17T08:25:49Z" message: InferenceGraph [switch-graph-54cf7] is Ready metadata: creationTimestamp: "2026-04-17T08:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T08:25:49Z" name: switch-graph-54cf7.18a71770601253be namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: a128072c-5968-488f-ae57-12f14ada7123 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:26:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "26231" uid: f4d50c78-0736-49a8-adb5-5a9a7f9b463c kind: Event lastTimestamp: "2026-04-17T08:26:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:26:19Z" name: switch-graph-54cf7.18a71773db5198b8 namespace: kserve-ci-e2e-test resourceVersion: "26520" uid: 28955e1c-13cc-460e-8636-24a62a579921 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T08:26:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-54cf7 namespace: kserve-ci-e2e-test resourceVersion: "26231" uid: f4d50c78-0736-49a8-adb5-5a9a7f9b463c kind: Event lastTimestamp: "2026-04-17T08:26:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T08:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T08:26:19Z" name: switch-graph-54cf7.18a71773db94c662 namespace: kserve-ci-e2e-test resourceVersion: "26521" uid: 93ed2b57-efbd-443f-96e2-14aeba91d580 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33389"